Compare commits
46 Commits
e6a2731d7e
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
59c3f3ee06 | ||
|
|
9c4c3096d7 | ||
|
|
b02bd6b66c | ||
|
|
890ac4c86e | ||
| 9c6e974d3a | |||
| d1dc32d3bb | |||
| a6a0accd6a | |||
| 3b8b1f5492 | |||
| 0c80a08732 | |||
| 8b72e62621 | |||
| ff7a4b69cd | |||
| e7f9594381 | |||
| 0c0f219d02 | |||
| 9cf0ed84d9 | |||
| 2702edce22 | |||
| c33533fcd6 | |||
| 32eb047551 | |||
| d27d64e407 | |||
| fa6893aa98 | |||
| 77a8265b40 | |||
| c692f8d653 | |||
| 47096c9877 | |||
| 0104532e73 | |||
| 19e7e48190 | |||
| 7ef3ffe62c | |||
| 1853ef6452 | |||
| 34ef9490a4 | |||
| f3bc3da9bb | |||
| f797234abd | |||
| c95a4286fb | |||
| b72f281651 | |||
| 98912c5b03 | |||
| 28876fa1d2 | |||
| 5f46d26037 | |||
| 57433c7e75 | |||
| 9e7d7b88ac | |||
| bd88eaa3c8 | |||
| cd4cbec7e0 | |||
| 144f6e7c86 | |||
| ef8f7cbeaa | |||
|
|
507da3165f | ||
|
|
f41792a0fa | ||
|
|
e16a02453e | ||
|
|
b46d7d85c6 | ||
|
|
e5543f408a | ||
|
|
b239f564b2 |
20
bootstrap/argocd/application-monitoring-extras.yaml
Normal file
20
bootstrap/argocd/application-monitoring-extras.yaml
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: monitoring-extras
|
||||||
|
namespace: argocd
|
||||||
|
spec:
|
||||||
|
project: default
|
||||||
|
source:
|
||||||
|
repoURL: https://gitea.mpabi.pl/trade/trade-deploy.git
|
||||||
|
targetRevision: main
|
||||||
|
path: kustomize/infra/monitoring-extras
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: monitoring
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
|
syncOptions:
|
||||||
|
- CreateNamespace=true
|
||||||
50
bootstrap/argocd/application-monitoring-stack.yaml
Normal file
50
bootstrap/argocd/application-monitoring-stack.yaml
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
apiVersion: argoproj.io/v1alpha1
|
||||||
|
kind: Application
|
||||||
|
metadata:
|
||||||
|
name: monitoring-stack
|
||||||
|
namespace: argocd
|
||||||
|
spec:
|
||||||
|
project: default
|
||||||
|
source:
|
||||||
|
repoURL: https://prometheus-community.github.io/helm-charts
|
||||||
|
chart: kube-prometheus-stack
|
||||||
|
targetRevision: 81.6.9
|
||||||
|
helm:
|
||||||
|
skipCrds: true
|
||||||
|
values: |
|
||||||
|
grafana:
|
||||||
|
enabled: true
|
||||||
|
prometheus:
|
||||||
|
prometheusSpec:
|
||||||
|
scrapeInterval: 15s
|
||||||
|
evaluationInterval: 15s
|
||||||
|
additionalScrapeConfigs:
|
||||||
|
- job_name: mpabi-yellowstone-geyser
|
||||||
|
metrics_path: /metrics
|
||||||
|
scrape_interval: 10s
|
||||||
|
fallback_scrape_protocol: PrometheusText0.0.4
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- 10.66.66.1:8999
|
||||||
|
- job_name: mpabi-node-exporter
|
||||||
|
metrics_path: /metrics
|
||||||
|
scrape_interval: 15s
|
||||||
|
static_configs:
|
||||||
|
- targets:
|
||||||
|
- 10.66.66.1:9100
|
||||||
|
prometheusOperator:
|
||||||
|
admissionWebhooks:
|
||||||
|
enabled: false
|
||||||
|
patch:
|
||||||
|
enabled: false
|
||||||
|
destination:
|
||||||
|
server: https://kubernetes.default.svc
|
||||||
|
namespace: monitoring
|
||||||
|
syncPolicy:
|
||||||
|
automated:
|
||||||
|
prune: true
|
||||||
|
selfHeal: true
|
||||||
|
syncOptions:
|
||||||
|
- CreateNamespace=true
|
||||||
|
- Replace=true
|
||||||
|
- ServerSideApply=true
|
||||||
@@ -18,6 +18,9 @@ spec:
|
|||||||
- name: trade-api-wrapper
|
- name: trade-api-wrapper
|
||||||
configMap:
|
configMap:
|
||||||
name: trade-api-wrapper
|
name: trade-api-wrapper
|
||||||
|
- name: trade-api-upstream
|
||||||
|
configMap:
|
||||||
|
name: trade-api-upstream
|
||||||
containers:
|
containers:
|
||||||
- name: api
|
- name: api
|
||||||
image: gitea.mpabi.pl/trade/trade-api:k3s-20260111095435
|
image: gitea.mpabi.pl/trade/trade-api:k3s-20260111095435
|
||||||
@@ -31,6 +34,10 @@ spec:
|
|||||||
mountPath: /override/wrapper.mjs
|
mountPath: /override/wrapper.mjs
|
||||||
subPath: wrapper.mjs
|
subPath: wrapper.mjs
|
||||||
readOnly: true
|
readOnly: true
|
||||||
|
- name: trade-api-upstream
|
||||||
|
mountPath: /app/services/api/server.mjs
|
||||||
|
subPath: server.mjs
|
||||||
|
readOnly: true
|
||||||
env:
|
env:
|
||||||
- name: PORT
|
- name: PORT
|
||||||
value: "8787"
|
value: "8787"
|
||||||
|
|||||||
1759
kustomize/base/api/server.mjs
Normal file
1759
kustomize/base/api/server.mjs
Normal file
File diff suppressed because it is too large
Load Diff
48
kustomize/base/candles-cache-worker/deployment.yaml
Normal file
48
kustomize/base/candles-cache-worker/deployment.yaml
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: candles-cache-worker
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "6"
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: candles-cache-worker
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: candles-cache-worker
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: worker
|
||||||
|
image: node:20-slim
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
env:
|
||||||
|
- name: HASURA_GRAPHQL_URL
|
||||||
|
value: http://hasura:8080/v1/graphql
|
||||||
|
- name: HASURA_ADMIN_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: trade-hasura
|
||||||
|
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||||
|
- name: CANDLES_SYMBOLS
|
||||||
|
value: SOL-PERP,PUMP-PERP
|
||||||
|
- name: CANDLES_SOURCES
|
||||||
|
value: any
|
||||||
|
- name: CANDLES_TFS
|
||||||
|
value: 1s,3s,5s,15s,30s,1m,3m,5m,15m,30m,1h,4h,12h,1d
|
||||||
|
- name: CANDLES_TARGET_POINTS
|
||||||
|
value: "1024"
|
||||||
|
- name: CANDLES_POLL_MS
|
||||||
|
value: "5000"
|
||||||
|
command: ["node", "/app/worker.mjs"]
|
||||||
|
volumeMounts:
|
||||||
|
- name: script
|
||||||
|
mountPath: /app/worker.mjs
|
||||||
|
subPath: worker.mjs
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: script
|
||||||
|
configMap:
|
||||||
|
name: candles-cache-worker-script
|
||||||
431
kustomize/base/candles-cache-worker/worker.mjs
Normal file
431
kustomize/base/candles-cache-worker/worker.mjs
Normal file
@@ -0,0 +1,431 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
|
import process from 'node:process';
|
||||||
|
import { setTimeout as sleep } from 'node:timers/promises';
|
||||||
|
|
||||||
|
function readJsonFile(filePath) {
|
||||||
|
try {
|
||||||
|
const raw = fs.readFileSync(filePath, 'utf8');
|
||||||
|
return JSON.parse(raw);
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function getIsoNow() {
|
||||||
|
return new Date().toISOString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function clampInt(value, min, max, fallback) {
|
||||||
|
const n = Number.parseInt(String(value ?? ''), 10);
|
||||||
|
if (!Number.isInteger(n)) return fallback;
|
||||||
|
return Math.min(max, Math.max(min, n));
|
||||||
|
}
|
||||||
|
|
||||||
|
function envList(name, fallbackCsv) {
|
||||||
|
const raw = process.env[name] ?? fallbackCsv;
|
||||||
|
return String(raw)
|
||||||
|
.split(',')
|
||||||
|
.map((s) => s.trim())
|
||||||
|
.filter(Boolean);
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseTimeframeToSeconds(tf) {
|
||||||
|
const v = String(tf || '').trim().toLowerCase();
|
||||||
|
if (!v) return 60;
|
||||||
|
const m = v.match(/^(\d+)(s|m|h|d)$/);
|
||||||
|
if (!m) throw new Error('invalid_tf');
|
||||||
|
const num = Number.parseInt(m[1], 10);
|
||||||
|
if (!Number.isInteger(num) || num <= 0) throw new Error('invalid_tf');
|
||||||
|
const unit = m[2];
|
||||||
|
const mult = unit === 's' ? 1 : unit === 'm' ? 60 : unit === 'h' ? 3600 : 86400;
|
||||||
|
return num * mult;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sqlLit(value) {
|
||||||
|
const s = String(value ?? '');
|
||||||
|
return `'${s.replace(/'/g, "''")}'`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveHasuraBaseUrl(graphqlUrlOrBase) {
|
||||||
|
const u = String(graphqlUrlOrBase || '').trim();
|
||||||
|
if (!u) return 'http://hasura:8080';
|
||||||
|
// common case: http://hasura:8080/v1/graphql
|
||||||
|
if (u.endsWith('/v1/graphql')) return u.slice(0, -'/v1/graphql'.length);
|
||||||
|
return u.replace(/\/$/, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveConfig() {
|
||||||
|
const tokensPath =
|
||||||
|
process.env.HASURA_TOKENS_FILE ||
|
||||||
|
process.env.TOKENS_FILE ||
|
||||||
|
process.env.HASURA_CONFIG_FILE ||
|
||||||
|
'/app/tokens/hasura.json';
|
||||||
|
const tokens = readJsonFile(tokensPath) || {};
|
||||||
|
|
||||||
|
const graphqlUrl =
|
||||||
|
process.env.HASURA_GRAPHQL_URL ||
|
||||||
|
tokens.graphqlUrl ||
|
||||||
|
tokens.apiUrl ||
|
||||||
|
'http://hasura:8080/v1/graphql';
|
||||||
|
const hasuraBaseUrl = resolveHasuraBaseUrl(graphqlUrl);
|
||||||
|
|
||||||
|
const hasuraAdminSecret =
|
||||||
|
process.env.HASURA_ADMIN_SECRET ||
|
||||||
|
process.env.HASURA_GRAPHQL_ADMIN_SECRET ||
|
||||||
|
tokens.adminSecret ||
|
||||||
|
tokens.hasuraAdminSecret;
|
||||||
|
|
||||||
|
if (!hasuraAdminSecret) throw new Error('Missing HASURA_ADMIN_SECRET (required for /v2/query run_sql)');
|
||||||
|
|
||||||
|
const symbols = envList('CANDLES_SYMBOLS', 'SOL-PERP,PUMP-PERP');
|
||||||
|
|
||||||
|
// sources: include "any" (sourceKey='') by default.
|
||||||
|
const rawSources = envList('CANDLES_SOURCES', 'any');
|
||||||
|
const sources = [];
|
||||||
|
for (const s of rawSources) {
|
||||||
|
const v = String(s).trim();
|
||||||
|
if (!v) continue;
|
||||||
|
if (v.toLowerCase() === 'any') sources.push('');
|
||||||
|
else sources.push(v);
|
||||||
|
}
|
||||||
|
if (!sources.includes('')) sources.unshift('');
|
||||||
|
|
||||||
|
const tfList = envList('CANDLES_TFS', '1s,3s,5s,15s,30s,1m,3m,5m,15m,30m,1h,4h,12h,1d');
|
||||||
|
const bucketSecondsList = tfList
|
||||||
|
.map((t) => {
|
||||||
|
try {
|
||||||
|
return parseTimeframeToSeconds(t);
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.filter((n) => n != null)
|
||||||
|
.map((n) => n)
|
||||||
|
.filter((n, idx, arr) => arr.findIndex((x) => x === n) === idx)
|
||||||
|
.sort((a, b) => a - b);
|
||||||
|
|
||||||
|
const targetPoints = clampInt(process.env.CANDLES_TARGET_POINTS, 10, 100_000, 1024);
|
||||||
|
// legacy: kept for compatibility; if set, used as a minimum warmup window (days).
|
||||||
|
const backfillDays = clampInt(process.env.CANDLES_BACKFILL_DAYS, 0, 3650, 0);
|
||||||
|
const pollMs = clampInt(process.env.CANDLES_POLL_MS, 250, 60_000, 5000);
|
||||||
|
|
||||||
|
return { hasuraBaseUrl, hasuraAdminSecret, symbols, sources, bucketSecondsList, targetPoints, backfillDays, pollMs };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function hasuraRunSql(cfg, sql, { readOnly } = { readOnly: false }) {
|
||||||
|
const url = `${cfg.hasuraBaseUrl}/v2/query`;
|
||||||
|
const body = {
|
||||||
|
type: 'run_sql',
|
||||||
|
args: {
|
||||||
|
source: 'default',
|
||||||
|
sql,
|
||||||
|
read_only: Boolean(readOnly),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
const res = await fetch(url, {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'content-type': 'application/json', 'x-hasura-admin-secret': cfg.hasuraAdminSecret },
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
signal: AbortSignal.timeout(60_000),
|
||||||
|
});
|
||||||
|
const text = await res.text();
|
||||||
|
if (!res.ok) throw new Error(`Hasura run_sql HTTP ${res.status}: ${text}`);
|
||||||
|
return JSON.parse(text);
|
||||||
|
}
|
||||||
|
|
||||||
|
function chunkSecondsForBucket(bucketSeconds) {
|
||||||
|
if (bucketSeconds <= 5) return 15 * 60;
|
||||||
|
if (bucketSeconds <= 60) return 60 * 60;
|
||||||
|
if (bucketSeconds <= 300) return 6 * 60 * 60;
|
||||||
|
if (bucketSeconds <= 3600) return 24 * 60 * 60;
|
||||||
|
return 7 * 24 * 60 * 60;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sqlUpsertCandlesFromTicks({ symbol, sourceKey, bucketSeconds, fromIso, toIso }) {
|
||||||
|
return `
|
||||||
|
WITH chosen AS (
|
||||||
|
SELECT source AS chosen_source
|
||||||
|
FROM public.drift_ticks
|
||||||
|
WHERE symbol = ${sqlLit(symbol)}
|
||||||
|
ORDER BY ts DESC
|
||||||
|
LIMIT 1
|
||||||
|
),
|
||||||
|
base AS (
|
||||||
|
SELECT
|
||||||
|
time_bucket(make_interval(secs => ${bucketSeconds}), ts) AS bucket,
|
||||||
|
ts,
|
||||||
|
COALESCE(mark_price, oracle_price) AS px,
|
||||||
|
oracle_price AS oracle_px
|
||||||
|
FROM public.drift_ticks
|
||||||
|
WHERE symbol = ${sqlLit(symbol)}
|
||||||
|
AND ts >= ${sqlLit(fromIso)}::timestamptz
|
||||||
|
AND ts < ${sqlLit(toIso)}::timestamptz
|
||||||
|
AND (
|
||||||
|
${sqlLit(sourceKey)} <> '' AND source = ${sqlLit(sourceKey)}
|
||||||
|
OR ${sqlLit(sourceKey)} = '' AND source = COALESCE((SELECT chosen_source FROM chosen), source)
|
||||||
|
)
|
||||||
|
),
|
||||||
|
agg AS (
|
||||||
|
SELECT
|
||||||
|
bucket,
|
||||||
|
(array_agg(px ORDER BY ts ASC))[1] AS open,
|
||||||
|
max(px) AS high,
|
||||||
|
min(px) AS low,
|
||||||
|
(array_agg(px ORDER BY ts DESC))[1] AS close,
|
||||||
|
(array_agg(oracle_px ORDER BY ts ASC))[1] AS oracle_open,
|
||||||
|
max(oracle_px) AS oracle_high,
|
||||||
|
min(oracle_px) AS oracle_low,
|
||||||
|
(array_agg(oracle_px ORDER BY ts DESC))[1] AS oracle_close,
|
||||||
|
count(*)::bigint AS ticks
|
||||||
|
FROM base
|
||||||
|
GROUP BY bucket
|
||||||
|
)
|
||||||
|
INSERT INTO public.drift_candles_cache
|
||||||
|
(
|
||||||
|
bucket,
|
||||||
|
bucket_seconds,
|
||||||
|
symbol,
|
||||||
|
source,
|
||||||
|
open,
|
||||||
|
high,
|
||||||
|
low,
|
||||||
|
close,
|
||||||
|
oracle_open,
|
||||||
|
oracle_high,
|
||||||
|
oracle_low,
|
||||||
|
oracle_close,
|
||||||
|
ticks,
|
||||||
|
updated_at
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
bucket,
|
||||||
|
${bucketSeconds},
|
||||||
|
${sqlLit(symbol)},
|
||||||
|
${sqlLit(sourceKey)},
|
||||||
|
open,
|
||||||
|
high,
|
||||||
|
low,
|
||||||
|
close,
|
||||||
|
oracle_open,
|
||||||
|
oracle_high,
|
||||||
|
oracle_low,
|
||||||
|
oracle_close,
|
||||||
|
ticks,
|
||||||
|
now()
|
||||||
|
FROM agg
|
||||||
|
ON CONFLICT (bucket, bucket_seconds, symbol, source) DO UPDATE SET
|
||||||
|
open = EXCLUDED.open,
|
||||||
|
high = EXCLUDED.high,
|
||||||
|
low = EXCLUDED.low,
|
||||||
|
close = EXCLUDED.close,
|
||||||
|
oracle_open = EXCLUDED.oracle_open,
|
||||||
|
oracle_high = EXCLUDED.oracle_high,
|
||||||
|
oracle_low = EXCLUDED.oracle_low,
|
||||||
|
oracle_close = EXCLUDED.oracle_close,
|
||||||
|
ticks = EXCLUDED.ticks,
|
||||||
|
updated_at = now();
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sqlDeleteOlderCandles({ symbol, sourceKey, bucketSeconds, cutoffIso }) {
|
||||||
|
return `
|
||||||
|
DELETE FROM public.drift_candles_cache
|
||||||
|
WHERE symbol = ${sqlLit(symbol)}
|
||||||
|
AND source = ${sqlLit(sourceKey)}
|
||||||
|
AND bucket_seconds = ${bucketSeconds}
|
||||||
|
AND bucket < ${sqlLit(cutoffIso)}::timestamptz;
|
||||||
|
`;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getTickRange(cfg, { symbol, sourceKey }) {
|
||||||
|
const sql =
|
||||||
|
String(sourceKey) === ''
|
||||||
|
? `
|
||||||
|
WITH chosen AS (
|
||||||
|
SELECT source AS chosen_source
|
||||||
|
FROM public.drift_ticks
|
||||||
|
WHERE symbol=${sqlLit(symbol)}
|
||||||
|
ORDER BY ts DESC
|
||||||
|
LIMIT 1
|
||||||
|
)
|
||||||
|
SELECT min(ts) AS min_ts, max(ts) AS max_ts
|
||||||
|
FROM public.drift_ticks
|
||||||
|
WHERE symbol=${sqlLit(symbol)}
|
||||||
|
AND source = COALESCE((SELECT chosen_source FROM chosen), source);
|
||||||
|
`
|
||||||
|
: `
|
||||||
|
SELECT min(ts) AS min_ts, max(ts) AS max_ts
|
||||||
|
FROM public.drift_ticks
|
||||||
|
WHERE symbol=${sqlLit(symbol)}
|
||||||
|
AND source = ${sqlLit(sourceKey)};
|
||||||
|
`;
|
||||||
|
const out = await hasuraRunSql(cfg, sql, { readOnly: true });
|
||||||
|
const row = Array.isArray(out?.result) && out.result.length >= 2 ? out.result[1] : null;
|
||||||
|
if (!row) return { minTs: null, maxTs: null };
|
||||||
|
const minTs = row[0] ? String(row[0]).trim() : null;
|
||||||
|
const maxTs = row[1] ? String(row[1]).trim() : null;
|
||||||
|
return { minTs: minTs && minTs.length ? minTs : null, maxTs: maxTs && maxTs.length ? maxTs : null };
|
||||||
|
}
|
||||||
|
|
||||||
|
function desiredFromIso({ minTsIso, maxTsIso, bucketSeconds, targetPoints, backfillDays }) {
|
||||||
|
const endMs = Date.parse(maxTsIso);
|
||||||
|
const minMs = minTsIso ? Date.parse(minTsIso) : null;
|
||||||
|
const wantSpanMs = targetPoints * bucketSeconds * 1000;
|
||||||
|
const wantFromMs = endMs - wantSpanMs;
|
||||||
|
const minFromMs = backfillDays > 0 ? endMs - backfillDays * 24 * 60 * 60 * 1000 : null;
|
||||||
|
let fromMs = wantFromMs;
|
||||||
|
if (minFromMs != null) fromMs = Math.min(fromMs, minFromMs);
|
||||||
|
if (minMs != null) fromMs = Math.max(fromMs, minMs);
|
||||||
|
return new Date(Math.max(0, fromMs)).toISOString();
|
||||||
|
}
|
||||||
|
|
||||||
|
function safetyWindowSeconds(bucketSeconds) {
|
||||||
|
if (bucketSeconds <= 60) return 10 * 60;
|
||||||
|
if (bucketSeconds <= 300) return 60 * 60;
|
||||||
|
if (bucketSeconds <= 3600) return 6 * 60 * 60;
|
||||||
|
if (bucketSeconds <= 14_400) return 24 * 60 * 60;
|
||||||
|
return 2 * 24 * 60 * 60;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function backfillBucket(cfg, { symbol, sourceKey, bucketSeconds, fromIso, toIso }) {
|
||||||
|
const chunk = chunkSecondsForBucket(bucketSeconds);
|
||||||
|
for (let t = Date.parse(fromIso); t < Date.parse(toIso); t += chunk * 1000) {
|
||||||
|
const a = new Date(t).toISOString();
|
||||||
|
const b = new Date(Math.min(Date.parse(toIso), t + chunk * 1000)).toISOString();
|
||||||
|
await hasuraRunSql(cfg, sqlUpsertCandlesFromTicks({ symbol, sourceKey, bucketSeconds, fromIso: a, toIso: b }));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function getMaxBucket(cfg, { symbol, sourceKey, bucketSeconds }) {
|
||||||
|
const sql = `
|
||||||
|
SELECT max(bucket) AS max_bucket
|
||||||
|
FROM public.drift_candles_cache
|
||||||
|
WHERE symbol=${sqlLit(symbol)} AND source=${sqlLit(sourceKey)} AND bucket_seconds=${bucketSeconds};
|
||||||
|
`;
|
||||||
|
const out = await hasuraRunSql(cfg, sql, { readOnly: true });
|
||||||
|
// Hasura returns {result_type, result:[[col...],[row...]]}
|
||||||
|
const row = Array.isArray(out?.result) && out.result.length >= 2 ? out.result[1] : null;
|
||||||
|
const v = row && row[0] ? String(row[0]) : null;
|
||||||
|
return v && v.trim() ? v.trim() : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const cfg = resolveConfig();
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
JSON.stringify(
|
||||||
|
{
|
||||||
|
service: 'candles-cache-worker',
|
||||||
|
startedAt: getIsoNow(),
|
||||||
|
hasuraBaseUrl: cfg.hasuraBaseUrl,
|
||||||
|
symbols: cfg.symbols,
|
||||||
|
sources: cfg.sources.map((s) => (s ? s : '(any)')),
|
||||||
|
bucketSecondsList: cfg.bucketSecondsList,
|
||||||
|
targetPoints: cfg.targetPoints,
|
||||||
|
backfillDays: cfg.backfillDays,
|
||||||
|
pollMs: cfg.pollMs,
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
// Backfill to warm cache: for each timeframe keep ~targetPoints candles (or "as much as we have").
|
||||||
|
for (const symbol of cfg.symbols) {
|
||||||
|
for (const sourceKey of cfg.sources) {
|
||||||
|
const range = await getTickRange(cfg, { symbol, sourceKey });
|
||||||
|
if (!range.maxTs) continue;
|
||||||
|
const toIso = new Date(Date.parse(range.maxTs)).toISOString();
|
||||||
|
for (const bs of cfg.bucketSecondsList) {
|
||||||
|
const fromIso = desiredFromIso({
|
||||||
|
minTsIso: range.minTs,
|
||||||
|
maxTsIso: toIso,
|
||||||
|
bucketSeconds: bs,
|
||||||
|
targetPoints: cfg.targetPoints,
|
||||||
|
backfillDays: cfg.backfillDays,
|
||||||
|
});
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`[candles-cache-worker] warmup symbol=${symbol} source=${sourceKey || '(any)'} bs=${bs}s from=${fromIso} to=${toIso} points=${cfg.targetPoints}`
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
await backfillBucket(cfg, { symbol, sourceKey, bucketSeconds: bs, fromIso, toIso });
|
||||||
|
// Enforce max window for this bucket (derived data; safe to prune).
|
||||||
|
await hasuraRunSql(cfg, sqlDeleteOlderCandles({ symbol, sourceKey, bucketSeconds: bs, cutoffIso: fromIso }));
|
||||||
|
} catch (err) {
|
||||||
|
console.error(
|
||||||
|
`[candles-cache-worker] warmup failed (${symbol}/${sourceKey || 'any'}/${bs}s): ${String(err?.message || err)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Prime last buckets.
|
||||||
|
const last = new Map(); // key -> iso bucket
|
||||||
|
for (const symbol of cfg.symbols) {
|
||||||
|
for (const sourceKey of cfg.sources) {
|
||||||
|
for (const bs of cfg.bucketSecondsList) {
|
||||||
|
const k = `${symbol}::${sourceKey}::${bs}`;
|
||||||
|
try {
|
||||||
|
const maxBucket = await getMaxBucket(cfg, { symbol, sourceKey, bucketSeconds: bs });
|
||||||
|
if (maxBucket) last.set(k, maxBucket);
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const loopNow = Date.now();
|
||||||
|
const loopIso = new Date(loopNow).toISOString();
|
||||||
|
const pruneEveryMs = 60_000;
|
||||||
|
const lastPruneAt = last.__pruneAt || new Map();
|
||||||
|
// stash on the Map to keep closure-local without introducing a new outer var
|
||||||
|
last.__pruneAt = lastPruneAt;
|
||||||
|
|
||||||
|
for (const symbol of cfg.symbols) {
|
||||||
|
for (const sourceKey of cfg.sources) {
|
||||||
|
for (const bs of cfg.bucketSecondsList) {
|
||||||
|
const k = `${symbol}::${sourceKey}::${bs}`;
|
||||||
|
const prev = last.get(k);
|
||||||
|
const safety = safetyWindowSeconds(bs);
|
||||||
|
const fromMs = prev ? Date.parse(prev) - safety * 1000 : loopNow - safety * 1000;
|
||||||
|
const fromIso2 = new Date(Math.max(0, fromMs)).toISOString();
|
||||||
|
try {
|
||||||
|
await hasuraRunSql(
|
||||||
|
cfg,
|
||||||
|
sqlUpsertCandlesFromTicks({ symbol, sourceKey, bucketSeconds: bs, fromIso: fromIso2, toIso: loopIso })
|
||||||
|
);
|
||||||
|
// best-effort: move last pointer close to now (actual max will lag by at most one bucket)
|
||||||
|
last.set(k, loopIso);
|
||||||
|
|
||||||
|
const prevPrune = lastPruneAt.get(k) || 0;
|
||||||
|
if (loopNow - prevPrune >= pruneEveryMs) {
|
||||||
|
const keepSeconds = Math.max(cfg.targetPoints * bs, (cfg.backfillDays > 0 ? cfg.backfillDays * 86400 : 0));
|
||||||
|
const cutoffIso = new Date(Math.max(0, loopNow - keepSeconds * 1000)).toISOString();
|
||||||
|
try {
|
||||||
|
await hasuraRunSql(cfg, sqlDeleteOlderCandles({ symbol, sourceKey, bucketSeconds: bs, cutoffIso }));
|
||||||
|
} finally {
|
||||||
|
lastPruneAt.set(k, loopNow);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.error(
|
||||||
|
`[candles-cache-worker] update failed (${symbol}/${sourceKey || 'any'}/${bs}s): ${String(err?.message || err)}`
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
await sleep(cfg.pollMs);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
main().catch((err) => {
|
||||||
|
console.error(String(err?.stack || err));
|
||||||
|
process.exitCode = 1;
|
||||||
|
});
|
||||||
50
kustomize/base/dlob-depth-worker/deployment-drift.yaml
Normal file
50
kustomize/base/dlob-depth-worker/deployment-drift.yaml
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: dlob-depth-worker-drift
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "6"
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: dlob-depth-worker-drift
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: dlob-depth-worker-drift
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: worker
|
||||||
|
image: node:20-slim
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
env:
|
||||||
|
- name: HASURA_GRAPHQL_URL
|
||||||
|
value: http://hasura:8080/v1/graphql
|
||||||
|
- name: HASURA_ADMIN_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: trade-hasura
|
||||||
|
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||||
|
- name: DLOB_SOURCE
|
||||||
|
value: drift
|
||||||
|
- name: DLOB_MARKETS
|
||||||
|
value: SOL-PERP,DOGE-PERP,JUP-PERP
|
||||||
|
- name: DLOB_POLL_MS
|
||||||
|
value: "1000"
|
||||||
|
- name: DLOB_DEPTH_BPS_BANDS
|
||||||
|
value: "5,10,20,50,100,200"
|
||||||
|
- name: PRICE_PRECISION
|
||||||
|
value: "1000000"
|
||||||
|
- name: BASE_PRECISION
|
||||||
|
value: "1000000000"
|
||||||
|
command: ["node", "/app/worker.mjs"]
|
||||||
|
volumeMounts:
|
||||||
|
- name: script
|
||||||
|
mountPath: /app/worker.mjs
|
||||||
|
subPath: worker.mjs
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: script
|
||||||
|
configMap:
|
||||||
|
name: dlob-depth-worker-script
|
||||||
@@ -26,8 +26,10 @@ spec:
|
|||||||
secretKeyRef:
|
secretKeyRef:
|
||||||
name: trade-hasura
|
name: trade-hasura
|
||||||
key: HASURA_GRAPHQL_ADMIN_SECRET
|
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||||
|
- name: DLOB_SOURCE
|
||||||
|
value: mevnode
|
||||||
- name: DLOB_MARKETS
|
- name: DLOB_MARKETS
|
||||||
value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
|
value: SOL-PERP,DOGE-PERP,JUP-PERP
|
||||||
- name: DLOB_POLL_MS
|
- name: DLOB_POLL_MS
|
||||||
value: "1000"
|
value: "1000"
|
||||||
- name: DLOB_DEPTH_BPS_BANDS
|
- name: DLOB_DEPTH_BPS_BANDS
|
||||||
|
|||||||
@@ -64,7 +64,8 @@ function resolveConfig() {
|
|||||||
const hasuraAdminSecret = process.env.HASURA_ADMIN_SECRET || process.env.HASURA_GRAPHQL_ADMIN_SECRET || undefined;
|
const hasuraAdminSecret = process.env.HASURA_ADMIN_SECRET || process.env.HASURA_GRAPHQL_ADMIN_SECRET || undefined;
|
||||||
const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined;
|
const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined;
|
||||||
|
|
||||||
const markets = envList('DLOB_MARKETS', 'PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP');
|
const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode';
|
||||||
|
const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
|
||||||
const pollMs = clampInt(process.env.DLOB_POLL_MS, 250, 60_000, 1000);
|
const pollMs = clampInt(process.env.DLOB_POLL_MS, 250, 60_000, 1000);
|
||||||
const bandsBps = envIntList('DLOB_DEPTH_BPS_BANDS', '5,10,20,50,100,200');
|
const bandsBps = envIntList('DLOB_DEPTH_BPS_BANDS', '5,10,20,50,100,200');
|
||||||
|
|
||||||
@@ -79,6 +80,7 @@ function resolveConfig() {
|
|||||||
hasuraUrl,
|
hasuraUrl,
|
||||||
hasuraAdminSecret,
|
hasuraAdminSecret,
|
||||||
hasuraAuthToken,
|
hasuraAuthToken,
|
||||||
|
dlobSource,
|
||||||
markets,
|
markets,
|
||||||
pollMs,
|
pollMs,
|
||||||
bandsBps,
|
bandsBps,
|
||||||
@@ -169,8 +171,9 @@ function computeBandDepth({ bids, asks, mid, bandBps }) {
|
|||||||
|
|
||||||
async function fetchL2Latest(cfg) {
|
async function fetchL2Latest(cfg) {
|
||||||
const query = `
|
const query = `
|
||||||
query DlobL2Latest($markets: [String!]!) {
|
query DlobL2Latest($source: String!, $markets: [String!]!) {
|
||||||
dlob_l2_latest(where: {market_name: {_in: $markets}}) {
|
dlob_l2_latest(where: {source: {_eq: $source}, market_name: {_in: $markets}}) {
|
||||||
|
source
|
||||||
market_name
|
market_name
|
||||||
market_type
|
market_type
|
||||||
market_index
|
market_index
|
||||||
@@ -186,7 +189,7 @@ async function fetchL2Latest(cfg) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
`;
|
`;
|
||||||
const data = await graphqlRequest(cfg, query, { markets: cfg.markets });
|
const data = await graphqlRequest(cfg, query, { source: cfg.dlobSource, markets: cfg.markets });
|
||||||
return Array.isArray(data?.dlob_l2_latest) ? data.dlob_l2_latest : [];
|
return Array.isArray(data?.dlob_l2_latest) ? data.dlob_l2_latest : [];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -232,6 +235,7 @@ async function main() {
|
|||||||
startedAt: getIsoNow(),
|
startedAt: getIsoNow(),
|
||||||
hasuraUrl: cfg.hasuraUrl,
|
hasuraUrl: cfg.hasuraUrl,
|
||||||
hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none',
|
hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none',
|
||||||
|
dlobSource: cfg.dlobSource,
|
||||||
markets: cfg.markets,
|
markets: cfg.markets,
|
||||||
pollMs: cfg.pollMs,
|
pollMs: cfg.pollMs,
|
||||||
bandsBps: cfg.bandsBps,
|
bandsBps: cfg.bandsBps,
|
||||||
@@ -268,6 +272,7 @@ async function main() {
|
|||||||
for (const bandBps of cfg.bandsBps) {
|
for (const bandBps of cfg.bandsBps) {
|
||||||
const d = computeBandDepth({ bids, asks, mid, bandBps });
|
const d = computeBandDepth({ bids, asks, mid, bandBps });
|
||||||
rows.push({
|
rows.push({
|
||||||
|
source: cfg.dlobSource,
|
||||||
market_name: market,
|
market_name: market,
|
||||||
band_bps: bandBps,
|
band_bps: bandBps,
|
||||||
market_type: l2.market_type ? String(l2.market_type) : 'perp',
|
market_type: l2.market_type ? String(l2.market_type) : 'perp',
|
||||||
|
|||||||
50
kustomize/base/dlob-slippage-worker/deployment-drift.yaml
Normal file
50
kustomize/base/dlob-slippage-worker/deployment-drift.yaml
Normal file
@@ -0,0 +1,50 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: dlob-slippage-worker-drift
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "6"
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: dlob-slippage-worker-drift
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: dlob-slippage-worker-drift
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: worker
|
||||||
|
image: node:20-slim
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
env:
|
||||||
|
- name: HASURA_GRAPHQL_URL
|
||||||
|
value: http://hasura:8080/v1/graphql
|
||||||
|
- name: HASURA_ADMIN_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: trade-hasura
|
||||||
|
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||||
|
- name: DLOB_SOURCE
|
||||||
|
value: drift
|
||||||
|
- name: DLOB_MARKETS
|
||||||
|
value: SOL-PERP,DOGE-PERP,JUP-PERP
|
||||||
|
- name: DLOB_POLL_MS
|
||||||
|
value: "1000"
|
||||||
|
- name: DLOB_SLIPPAGE_SIZES_USD
|
||||||
|
value: "0.1,0.2,0.5,1,2,5,10,25,50,100,250,500,1000,5000,10000,50000"
|
||||||
|
- name: PRICE_PRECISION
|
||||||
|
value: "1000000"
|
||||||
|
- name: BASE_PRECISION
|
||||||
|
value: "1000000000"
|
||||||
|
command: ["node", "/app/worker.mjs"]
|
||||||
|
volumeMounts:
|
||||||
|
- name: script
|
||||||
|
mountPath: /app/worker.mjs
|
||||||
|
subPath: worker.mjs
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: script
|
||||||
|
configMap:
|
||||||
|
name: dlob-slippage-worker-script
|
||||||
@@ -26,12 +26,14 @@ spec:
|
|||||||
secretKeyRef:
|
secretKeyRef:
|
||||||
name: trade-hasura
|
name: trade-hasura
|
||||||
key: HASURA_GRAPHQL_ADMIN_SECRET
|
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||||
|
- name: DLOB_SOURCE
|
||||||
|
value: mevnode
|
||||||
- name: DLOB_MARKETS
|
- name: DLOB_MARKETS
|
||||||
value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
|
value: SOL-PERP,DOGE-PERP,JUP-PERP
|
||||||
- name: DLOB_POLL_MS
|
- name: DLOB_POLL_MS
|
||||||
value: "1000"
|
value: "1000"
|
||||||
- name: DLOB_SLIPPAGE_SIZES_USD
|
- name: DLOB_SLIPPAGE_SIZES_USD
|
||||||
value: "10,25,50,100,250,500,1000,5000,10000,50000"
|
value: "0.1,0.2,0.5,1,2,5,10,25,50,100,250,500,1000,5000,10000,50000"
|
||||||
- name: PRICE_PRECISION
|
- name: PRICE_PRECISION
|
||||||
value: "1000000"
|
value: "1000000"
|
||||||
- name: BASE_PRECISION
|
- name: BASE_PRECISION
|
||||||
|
|||||||
@@ -1,6 +1,16 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
import process from 'node:process';
|
import process from 'node:process';
|
||||||
import { setTimeout as sleep } from 'node:timers/promises';
|
import { setTimeout as sleep } from 'node:timers/promises';
|
||||||
|
|
||||||
|
function readJsonFile(filePath) {
|
||||||
|
try {
|
||||||
|
const raw = fs.readFileSync(filePath, 'utf8');
|
||||||
|
return JSON.parse(raw);
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function getIsoNow() {
|
function getIsoNow() {
|
||||||
return new Date().toISOString();
|
return new Date().toISOString();
|
||||||
}
|
}
|
||||||
@@ -19,14 +29,89 @@ function envList(name, fallbackCsv) {
|
|||||||
.filter(Boolean);
|
.filter(Boolean);
|
||||||
}
|
}
|
||||||
|
|
||||||
function envIntList(name, fallbackCsv) {
|
function parsePositiveNumber(value) {
|
||||||
const out = [];
|
const n = Number.parseFloat(String(value ?? '').trim());
|
||||||
for (const item of envList(name, fallbackCsv)) {
|
if (!Number.isFinite(n) || !(n > 0)) return null;
|
||||||
const n = Number.parseInt(item, 10);
|
return n;
|
||||||
if (!Number.isFinite(n)) continue;
|
|
||||||
out.push(n);
|
|
||||||
}
|
}
|
||||||
return out.length ? out : envList(name, fallbackCsv).map((v) => Number.parseInt(v, 10)).filter(Number.isFinite);
|
|
||||||
|
function resolveConfig() {
|
||||||
|
const tokensPath =
|
||||||
|
process.env.HASURA_TOKENS_FILE ||
|
||||||
|
process.env.TOKENS_FILE ||
|
||||||
|
process.env.HASURA_CONFIG_FILE ||
|
||||||
|
'/app/tokens/hasura.json';
|
||||||
|
const tokens = readJsonFile(tokensPath) || {};
|
||||||
|
|
||||||
|
const hasuraUrl =
|
||||||
|
process.env.HASURA_GRAPHQL_URL ||
|
||||||
|
tokens.graphqlUrl ||
|
||||||
|
tokens.apiUrl ||
|
||||||
|
'http://hasura:8080/v1/graphql';
|
||||||
|
const hasuraAdminSecret =
|
||||||
|
process.env.HASURA_ADMIN_SECRET ||
|
||||||
|
process.env.HASURA_GRAPHQL_ADMIN_SECRET ||
|
||||||
|
tokens.adminSecret ||
|
||||||
|
tokens.hasuraAdminSecret;
|
||||||
|
const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined;
|
||||||
|
|
||||||
|
const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode';
|
||||||
|
const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
|
||||||
|
const pollMs = clampInt(process.env.DLOB_POLL_MS, 250, 60_000, 1000);
|
||||||
|
|
||||||
|
const sizesUsd = envList('DLOB_SLIPPAGE_SIZES_USD', '10,25,50,100,250,500,1000')
|
||||||
|
.map(parsePositiveNumber)
|
||||||
|
.filter((n) => n != null)
|
||||||
|
.map((n) => n)
|
||||||
|
.filter((n, idx, arr) => arr.findIndex((x) => x === n) === idx)
|
||||||
|
.sort((a, b) => a - b);
|
||||||
|
|
||||||
|
const sizesUsdInt = sizesUsd.filter((n) => Number.isInteger(n));
|
||||||
|
|
||||||
|
const depthLevels = clampInt(process.env.DLOB_DEPTH, 1, 50, 25);
|
||||||
|
const pricePrecision = Number(process.env.PRICE_PRECISION || 1_000_000);
|
||||||
|
const basePrecision = Number(process.env.BASE_PRECISION || 1_000_000_000);
|
||||||
|
if (!Number.isFinite(pricePrecision) || pricePrecision <= 0) throw new Error(`Invalid PRICE_PRECISION: ${process.env.PRICE_PRECISION}`);
|
||||||
|
if (!Number.isFinite(basePrecision) || basePrecision <= 0) throw new Error(`Invalid BASE_PRECISION: ${process.env.BASE_PRECISION}`);
|
||||||
|
|
||||||
|
return {
|
||||||
|
hasuraUrl,
|
||||||
|
hasuraAdminSecret,
|
||||||
|
hasuraAuthToken,
|
||||||
|
dlobSource,
|
||||||
|
markets,
|
||||||
|
pollMs,
|
||||||
|
sizesUsd,
|
||||||
|
sizesUsdInt,
|
||||||
|
depthLevels,
|
||||||
|
pricePrecision,
|
||||||
|
basePrecision,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function graphqlRequest(cfg, query, variables) {
|
||||||
|
const headers = { 'content-type': 'application/json' };
|
||||||
|
if (cfg.hasuraAuthToken) {
|
||||||
|
headers.authorization = `Bearer ${cfg.hasuraAuthToken}`;
|
||||||
|
} else if (cfg.hasuraAdminSecret) {
|
||||||
|
headers['x-hasura-admin-secret'] = cfg.hasuraAdminSecret;
|
||||||
|
} else {
|
||||||
|
throw new Error('Missing Hasura auth (set HASURA_AUTH_TOKEN or HASURA_ADMIN_SECRET or mount tokens/hasura.json)');
|
||||||
|
}
|
||||||
|
|
||||||
|
const res = await fetch(cfg.hasuraUrl, {
|
||||||
|
method: 'POST',
|
||||||
|
headers,
|
||||||
|
body: JSON.stringify({ query, variables }),
|
||||||
|
signal: AbortSignal.timeout(15_000),
|
||||||
|
});
|
||||||
|
const text = await res.text();
|
||||||
|
if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`);
|
||||||
|
const json = JSON.parse(text);
|
||||||
|
if (json.errors?.length) {
|
||||||
|
throw new Error(json.errors.map((e) => e.message).join(' | '));
|
||||||
|
}
|
||||||
|
return json.data;
|
||||||
}
|
}
|
||||||
|
|
||||||
function toNumberOrNull(value) {
|
function toNumberOrNull(value) {
|
||||||
@@ -41,177 +126,117 @@ function toNumberOrNull(value) {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
function numStr(value) {
|
function normalizeLevels(raw) {
|
||||||
if (value == null) return null;
|
if (raw == null) return [];
|
||||||
if (typeof value === 'number') return Number.isFinite(value) ? String(value) : null;
|
if (Array.isArray(raw)) return raw;
|
||||||
if (typeof value === 'string') return value.trim() || null;
|
if (typeof raw === 'string') {
|
||||||
return null;
|
const s = raw.trim();
|
||||||
}
|
if (!s) return [];
|
||||||
|
|
||||||
function jsonNormalize(value) {
|
|
||||||
if (typeof value !== 'string') return value;
|
|
||||||
const s = value.trim();
|
|
||||||
if (!s) return null;
|
|
||||||
try {
|
try {
|
||||||
return JSON.parse(s);
|
const v = JSON.parse(s);
|
||||||
|
return Array.isArray(v) ? v : [];
|
||||||
} catch {
|
} catch {
|
||||||
return value;
|
return [];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return [];
|
||||||
function resolveConfig() {
|
|
||||||
const hasuraUrl = process.env.HASURA_GRAPHQL_URL || 'http://hasura:8080/v1/graphql';
|
|
||||||
const hasuraAdminSecret = process.env.HASURA_ADMIN_SECRET || process.env.HASURA_GRAPHQL_ADMIN_SECRET || undefined;
|
|
||||||
const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined;
|
|
||||||
|
|
||||||
const markets = envList('DLOB_MARKETS', 'PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP');
|
|
||||||
const pollMs = clampInt(process.env.DLOB_POLL_MS, 250, 60_000, 1000);
|
|
||||||
const sizesUsd = envIntList('DLOB_SLIPPAGE_SIZES_USD', '100,500,1000,5000,10000,50000');
|
|
||||||
|
|
||||||
const pricePrecision = Number(process.env.PRICE_PRECISION || 1_000_000);
|
|
||||||
const basePrecision = Number(process.env.BASE_PRECISION || 1_000_000_000);
|
|
||||||
if (!Number.isFinite(pricePrecision) || pricePrecision <= 0)
|
|
||||||
throw new Error(`Invalid PRICE_PRECISION: ${process.env.PRICE_PRECISION}`);
|
|
||||||
if (!Number.isFinite(basePrecision) || basePrecision <= 0)
|
|
||||||
throw new Error(`Invalid BASE_PRECISION: ${process.env.BASE_PRECISION}`);
|
|
||||||
|
|
||||||
return {
|
|
||||||
hasuraUrl,
|
|
||||||
hasuraAdminSecret,
|
|
||||||
hasuraAuthToken,
|
|
||||||
markets,
|
|
||||||
pollMs,
|
|
||||||
sizesUsd,
|
|
||||||
pricePrecision,
|
|
||||||
basePrecision,
|
|
||||||
};
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function graphqlRequest(cfg, query, variables) {
|
function parseScaledLevels(raw, pricePrecision, basePrecision) {
|
||||||
const headers = { 'content-type': 'application/json' };
|
const levels = normalizeLevels(raw);
|
||||||
if (cfg.hasuraAuthToken) {
|
|
||||||
headers.authorization = `Bearer ${cfg.hasuraAuthToken}`;
|
|
||||||
} else if (cfg.hasuraAdminSecret) {
|
|
||||||
headers['x-hasura-admin-secret'] = cfg.hasuraAdminSecret;
|
|
||||||
} else {
|
|
||||||
throw new Error('Missing Hasura auth (set HASURA_AUTH_TOKEN or HASURA_ADMIN_SECRET)');
|
|
||||||
}
|
|
||||||
|
|
||||||
const res = await fetch(cfg.hasuraUrl, {
|
|
||||||
method: 'POST',
|
|
||||||
headers,
|
|
||||||
body: JSON.stringify({ query, variables }),
|
|
||||||
signal: AbortSignal.timeout(10_000),
|
|
||||||
});
|
|
||||||
const text = await res.text();
|
|
||||||
if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`);
|
|
||||||
const json = JSON.parse(text);
|
|
||||||
if (json.errors?.length) throw new Error(json.errors.map((e) => e.message).join(' | '));
|
|
||||||
return json.data;
|
|
||||||
}
|
|
||||||
|
|
||||||
function parseLevels(raw, pricePrecision, basePrecision, side) {
|
|
||||||
const v = jsonNormalize(raw);
|
|
||||||
if (!Array.isArray(v)) return [];
|
|
||||||
|
|
||||||
const out = [];
|
const out = [];
|
||||||
for (const item of v) {
|
for (const it of levels) {
|
||||||
const priceInt = toNumberOrNull(item?.price);
|
const priceInt = toNumberOrNull(it?.price);
|
||||||
const sizeInt = toNumberOrNull(item?.size);
|
const sizeInt = toNumberOrNull(it?.size);
|
||||||
if (priceInt == null || sizeInt == null) continue;
|
if (priceInt == null || sizeInt == null) continue;
|
||||||
const price = priceInt / pricePrecision;
|
const price = priceInt / pricePrecision;
|
||||||
const size = sizeInt / basePrecision;
|
const base = sizeInt / basePrecision;
|
||||||
if (!Number.isFinite(price) || !Number.isFinite(size)) continue;
|
if (!Number.isFinite(price) || !Number.isFinite(base)) continue;
|
||||||
out.push({ price, size });
|
out.push({ price, base });
|
||||||
}
|
}
|
||||||
|
|
||||||
if (side === 'bid') out.sort((a, b) => b.price - a.price);
|
|
||||||
if (side === 'ask') out.sort((a, b) => a.price - b.price);
|
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
function computeMid(bestBid, bestAsk, markPrice, oraclePrice) {
|
function simulateFill(levels, sizeUsd) {
|
||||||
if (bestBid != null && bestAsk != null) return (bestBid + bestAsk) / 2;
|
|
||||||
if (markPrice != null) return markPrice;
|
|
||||||
if (oraclePrice != null) return oraclePrice;
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
function simulateBuy(asks, mid, sizeUsd) {
|
|
||||||
let remainingUsd = sizeUsd;
|
let remainingUsd = sizeUsd;
|
||||||
let filledUsd = 0;
|
let filledUsd = 0;
|
||||||
let filledBase = 0;
|
let filledBase = 0;
|
||||||
|
let totalQuoteUsd = 0;
|
||||||
let worstPrice = null;
|
let worstPrice = null;
|
||||||
let levelsConsumed = 0;
|
let levelsConsumed = 0;
|
||||||
|
|
||||||
for (const lvl of asks) {
|
for (const l of levels) {
|
||||||
if (!(remainingUsd > 0)) break;
|
if (remainingUsd <= 0) break;
|
||||||
if (!(lvl.price > 0) || !(lvl.size > 0)) continue;
|
const levelUsd = l.base * l.price;
|
||||||
|
if (levelUsd <= 0) continue;
|
||||||
const maxBase = remainingUsd / lvl.price;
|
|
||||||
const takeBase = Math.min(lvl.size, maxBase);
|
|
||||||
if (!(takeBase > 0)) continue;
|
|
||||||
|
|
||||||
const cost = takeBase * lvl.price;
|
|
||||||
filledUsd += cost;
|
|
||||||
filledBase += takeBase;
|
|
||||||
remainingUsd -= cost;
|
|
||||||
worstPrice = lvl.price;
|
|
||||||
levelsConsumed += 1;
|
levelsConsumed += 1;
|
||||||
}
|
worstPrice = l.price;
|
||||||
|
|
||||||
const vwap = filledBase > 0 ? filledUsd / filledBase : null;
|
const takeUsd = Math.min(remainingUsd, levelUsd);
|
||||||
const impactBps = vwap != null && mid != null && mid > 0 ? (vwap / mid - 1) * 10_000 : null;
|
const takeBase = takeUsd / l.price;
|
||||||
const fillPct = sizeUsd > 0 ? filledUsd / sizeUsd : null;
|
|
||||||
|
|
||||||
return { vwap, worstPrice, filledUsd, filledBase, impactBps, levelsConsumed, fillPct };
|
remainingUsd -= takeUsd;
|
||||||
}
|
filledUsd += takeUsd;
|
||||||
|
|
||||||
function simulateSell(bids, mid, sizeUsd) {
|
|
||||||
if (mid == null || !(mid > 0)) {
|
|
||||||
return { vwap: null, worstPrice: null, filledUsd: 0, filledBase: 0, impactBps: null, levelsConsumed: 0, fillPct: null };
|
|
||||||
}
|
|
||||||
|
|
||||||
const baseTarget = sizeUsd / mid;
|
|
||||||
let remainingBase = baseTarget;
|
|
||||||
let proceedsUsd = 0;
|
|
||||||
let filledBase = 0;
|
|
||||||
let worstPrice = null;
|
|
||||||
let levelsConsumed = 0;
|
|
||||||
|
|
||||||
for (const lvl of bids) {
|
|
||||||
if (!(remainingBase > 0)) break;
|
|
||||||
if (!(lvl.price > 0) || !(lvl.size > 0)) continue;
|
|
||||||
|
|
||||||
const takeBase = Math.min(lvl.size, remainingBase);
|
|
||||||
if (!(takeBase > 0)) continue;
|
|
||||||
|
|
||||||
const proceeds = takeBase * lvl.price;
|
|
||||||
proceedsUsd += proceeds;
|
|
||||||
filledBase += takeBase;
|
filledBase += takeBase;
|
||||||
remainingBase -= takeBase;
|
totalQuoteUsd += takeUsd;
|
||||||
worstPrice = lvl.price;
|
|
||||||
levelsConsumed += 1;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
const vwap = filledBase > 0 ? proceedsUsd / filledBase : null;
|
const vwapPrice = filledBase > 0 ? totalQuoteUsd / filledBase : null;
|
||||||
const impactBps = vwap != null && mid > 0 ? (1 - vwap / mid) * 10_000 : null;
|
const fillPct = sizeUsd > 0 ? (filledUsd / sizeUsd) * 100 : null;
|
||||||
const fillPct = baseTarget > 0 ? filledBase / baseTarget : null;
|
|
||||||
|
|
||||||
return { vwap, worstPrice, filledUsd: proceedsUsd, filledBase, impactBps, levelsConsumed, fillPct };
|
return {
|
||||||
|
filledUsd,
|
||||||
|
filledBase,
|
||||||
|
vwapPrice,
|
||||||
|
worstPrice,
|
||||||
|
levelsConsumed,
|
||||||
|
fillPct,
|
||||||
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async function fetchL2Latest(cfg) {
|
function impactBps({ side, mid, vwap }) {
|
||||||
|
if (mid == null || vwap == null || mid <= 0) return null;
|
||||||
|
if (side === 'buy') return ((vwap / mid) - 1) * 10_000;
|
||||||
|
if (side === 'sell') return (1 - (vwap / mid)) * 10_000;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function main() {
|
||||||
|
const cfg = resolveConfig();
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
JSON.stringify(
|
||||||
|
{
|
||||||
|
service: 'dlob-slippage-worker',
|
||||||
|
startedAt: getIsoNow(),
|
||||||
|
hasuraUrl: cfg.hasuraUrl,
|
||||||
|
hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none',
|
||||||
|
dlobSource: cfg.dlobSource,
|
||||||
|
markets: cfg.markets,
|
||||||
|
pollMs: cfg.pollMs,
|
||||||
|
sizesUsd: cfg.sizesUsd,
|
||||||
|
depthLevels: cfg.depthLevels,
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)
|
||||||
|
);
|
||||||
|
|
||||||
|
const lastSeenUpdatedAt = new Map(); // market -> updated_at
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
const updatedAt = getIsoNow();
|
||||||
|
|
||||||
|
try {
|
||||||
const query = `
|
const query = `
|
||||||
query DlobL2Latest($markets: [String!]!) {
|
query DlobL2Latest($source: String!, $markets: [String!]!) {
|
||||||
dlob_l2_latest(where: {market_name: {_in: $markets}}) {
|
dlob_l2_latest(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
|
||||||
|
source
|
||||||
market_name
|
market_name
|
||||||
market_type
|
market_type
|
||||||
market_index
|
market_index
|
||||||
ts
|
ts
|
||||||
slot
|
slot
|
||||||
mark_price
|
|
||||||
oracle_price
|
|
||||||
best_bid_price
|
best_bid_price
|
||||||
best_ask_price
|
best_ask_price
|
||||||
bids
|
bids
|
||||||
@@ -220,14 +245,95 @@ async function fetchL2Latest(cfg) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
`;
|
`;
|
||||||
const data = await graphqlRequest(cfg, query, { markets: cfg.markets });
|
|
||||||
return Array.isArray(data?.dlob_l2_latest) ? data.dlob_l2_latest : [];
|
const data = await graphqlRequest(cfg, query, { source: cfg.dlobSource, markets: cfg.markets });
|
||||||
|
const rows = Array.isArray(data?.dlob_l2_latest) ? data.dlob_l2_latest : [];
|
||||||
|
|
||||||
|
const objectsV1 = [];
|
||||||
|
const objectsV2 = [];
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
const market = String(row?.market_name || '').trim();
|
||||||
|
if (!market) continue;
|
||||||
|
|
||||||
|
const rowUpdatedAt = row?.updated_at ?? null;
|
||||||
|
if (rowUpdatedAt && lastSeenUpdatedAt.get(market) === rowUpdatedAt) continue;
|
||||||
|
if (rowUpdatedAt) lastSeenUpdatedAt.set(market, rowUpdatedAt);
|
||||||
|
|
||||||
|
const bestBid = toNumberOrNull(row?.best_bid_price);
|
||||||
|
const bestAsk = toNumberOrNull(row?.best_ask_price);
|
||||||
|
if (bestBid == null || bestAsk == null) continue;
|
||||||
|
|
||||||
|
const mid = (bestBid + bestAsk) / 2;
|
||||||
|
if (!Number.isFinite(mid) || mid <= 0) continue;
|
||||||
|
|
||||||
|
const bids = parseScaledLevels(row?.bids, cfg.pricePrecision, cfg.basePrecision)
|
||||||
|
.slice()
|
||||||
|
.sort((a, b) => b.price - a.price)
|
||||||
|
.slice(0, cfg.depthLevels);
|
||||||
|
const asks = parseScaledLevels(row?.asks, cfg.pricePrecision, cfg.basePrecision)
|
||||||
|
.slice()
|
||||||
|
.sort((a, b) => a.price - b.price)
|
||||||
|
.slice(0, cfg.depthLevels);
|
||||||
|
|
||||||
|
for (const sizeUsd of cfg.sizesUsd) {
|
||||||
|
// buy consumes asks (worse prices as you go up)
|
||||||
|
{
|
||||||
|
const sim = simulateFill(asks, sizeUsd);
|
||||||
|
const baseObj = {
|
||||||
|
source: cfg.dlobSource,
|
||||||
|
market_name: market,
|
||||||
|
side: 'buy',
|
||||||
|
market_type: row?.market_type ?? 'perp',
|
||||||
|
market_index: row?.market_index ?? null,
|
||||||
|
ts: row?.ts == null ? null : String(row.ts),
|
||||||
|
slot: row?.slot == null ? null : String(row.slot),
|
||||||
|
mid_price: String(mid),
|
||||||
|
vwap_price: sim.vwapPrice == null ? null : String(sim.vwapPrice),
|
||||||
|
worst_price: sim.worstPrice == null ? null : String(sim.worstPrice),
|
||||||
|
filled_usd: String(sim.filledUsd),
|
||||||
|
filled_base: String(sim.filledBase),
|
||||||
|
impact_bps: impactBps({ side: 'buy', mid, vwap: sim.vwapPrice }),
|
||||||
|
levels_consumed: sim.levelsConsumed,
|
||||||
|
fill_pct: sim.fillPct == null ? null : String(sim.fillPct),
|
||||||
|
raw: { depthLevels: cfg.depthLevels },
|
||||||
|
updated_at: updatedAt,
|
||||||
|
};
|
||||||
|
objectsV2.push({ ...baseObj, size_usd: String(sizeUsd) });
|
||||||
|
if (Number.isInteger(sizeUsd)) objectsV1.push({ ...baseObj, size_usd: Math.trunc(sizeUsd) });
|
||||||
}
|
}
|
||||||
|
|
||||||
async function upsertSlippage(cfg, rows) {
|
// sell consumes bids (worse prices as you go down)
|
||||||
if (!rows.length) return;
|
{
|
||||||
|
const sim = simulateFill(bids, sizeUsd);
|
||||||
|
const baseObj = {
|
||||||
|
source: cfg.dlobSource,
|
||||||
|
market_name: market,
|
||||||
|
side: 'sell',
|
||||||
|
market_type: row?.market_type ?? 'perp',
|
||||||
|
market_index: row?.market_index ?? null,
|
||||||
|
ts: row?.ts == null ? null : String(row.ts),
|
||||||
|
slot: row?.slot == null ? null : String(row.slot),
|
||||||
|
mid_price: String(mid),
|
||||||
|
vwap_price: sim.vwapPrice == null ? null : String(sim.vwapPrice),
|
||||||
|
worst_price: sim.worstPrice == null ? null : String(sim.worstPrice),
|
||||||
|
filled_usd: String(sim.filledUsd),
|
||||||
|
filled_base: String(sim.filledBase),
|
||||||
|
impact_bps: impactBps({ side: 'sell', mid, vwap: sim.vwapPrice }),
|
||||||
|
levels_consumed: sim.levelsConsumed,
|
||||||
|
fill_pct: sim.fillPct == null ? null : String(sim.fillPct),
|
||||||
|
raw: { depthLevels: cfg.depthLevels },
|
||||||
|
updated_at: updatedAt,
|
||||||
|
};
|
||||||
|
objectsV2.push({ ...baseObj, size_usd: String(sizeUsd) });
|
||||||
|
if (Number.isInteger(sizeUsd)) objectsV1.push({ ...baseObj, size_usd: Math.trunc(sizeUsd) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (objectsV1.length) {
|
||||||
const mutation = `
|
const mutation = `
|
||||||
mutation UpsertDlobSlippage($rows: [dlob_slippage_latest_insert_input!]!) {
|
mutation UpsertSlippageV1($rows: [dlob_slippage_latest_insert_input!]!) {
|
||||||
insert_dlob_slippage_latest(
|
insert_dlob_slippage_latest(
|
||||||
objects: $rows
|
objects: $rows
|
||||||
on_conflict: {
|
on_conflict: {
|
||||||
@@ -238,8 +344,6 @@ async function upsertSlippage(cfg, rows) {
|
|||||||
ts
|
ts
|
||||||
slot
|
slot
|
||||||
mid_price
|
mid_price
|
||||||
best_bid_price
|
|
||||||
best_ask_price
|
|
||||||
vwap_price
|
vwap_price
|
||||||
worst_price
|
worst_price
|
||||||
filled_usd
|
filled_usd
|
||||||
@@ -254,119 +358,40 @@ async function upsertSlippage(cfg, rows) {
|
|||||||
) { affected_rows }
|
) { affected_rows }
|
||||||
}
|
}
|
||||||
`;
|
`;
|
||||||
await graphqlRequest(cfg, mutation, { rows });
|
await graphqlRequest(cfg, mutation, { rows: objectsV1 });
|
||||||
}
|
}
|
||||||
|
|
||||||
async function main() {
|
if (objectsV2.length) {
|
||||||
const cfg = resolveConfig();
|
const mutation = `
|
||||||
const lastUpdatedAtByMarket = new Map();
|
mutation UpsertSlippageV2($rows: [dlob_slippage_latest_v2_insert_input!]!) {
|
||||||
|
insert_dlob_slippage_latest_v2(
|
||||||
console.log(
|
objects: $rows
|
||||||
JSON.stringify(
|
on_conflict: {
|
||||||
{
|
constraint: dlob_slippage_latest_v2_pkey
|
||||||
service: 'dlob-slippage-worker',
|
update_columns: [
|
||||||
startedAt: getIsoNow(),
|
market_type
|
||||||
hasuraUrl: cfg.hasuraUrl,
|
market_index
|
||||||
hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none',
|
ts
|
||||||
markets: cfg.markets,
|
slot
|
||||||
pollMs: cfg.pollMs,
|
mid_price
|
||||||
sizesUsd: cfg.sizesUsd,
|
vwap_price
|
||||||
pricePrecision: cfg.pricePrecision,
|
worst_price
|
||||||
basePrecision: cfg.basePrecision,
|
filled_usd
|
||||||
},
|
filled_base
|
||||||
null,
|
impact_bps
|
||||||
2
|
levels_consumed
|
||||||
)
|
fill_pct
|
||||||
);
|
raw
|
||||||
|
updated_at
|
||||||
while (true) {
|
]
|
||||||
const rows = [];
|
|
||||||
|
|
||||||
try {
|
|
||||||
const l2Rows = await fetchL2Latest(cfg);
|
|
||||||
for (const l2 of l2Rows) {
|
|
||||||
const market = String(l2.market_name || '').trim();
|
|
||||||
if (!market) continue;
|
|
||||||
|
|
||||||
const updatedAt = l2.updated_at || null;
|
|
||||||
if (updatedAt && lastUpdatedAtByMarket.get(market) === updatedAt) continue;
|
|
||||||
if (updatedAt) lastUpdatedAtByMarket.set(market, updatedAt);
|
|
||||||
|
|
||||||
const bestBid = toNumberOrNull(l2.best_bid_price);
|
|
||||||
const bestAsk = toNumberOrNull(l2.best_ask_price);
|
|
||||||
const markPrice = toNumberOrNull(l2.mark_price);
|
|
||||||
const oraclePrice = toNumberOrNull(l2.oracle_price);
|
|
||||||
const mid = computeMid(bestBid, bestAsk, markPrice, oraclePrice);
|
|
||||||
|
|
||||||
const bids = parseLevels(l2.bids, cfg.pricePrecision, cfg.basePrecision, 'bid');
|
|
||||||
const asks = parseLevels(l2.asks, cfg.pricePrecision, cfg.basePrecision, 'ask');
|
|
||||||
|
|
||||||
for (const sizeUsd of cfg.sizesUsd) {
|
|
||||||
const buy = simulateBuy(asks, mid, sizeUsd);
|
|
||||||
rows.push({
|
|
||||||
market_name: market,
|
|
||||||
side: 'buy',
|
|
||||||
size_usd: sizeUsd,
|
|
||||||
market_type: l2.market_type ? String(l2.market_type) : 'perp',
|
|
||||||
market_index: typeof l2.market_index === 'number' ? l2.market_index : null,
|
|
||||||
ts: l2.ts == null ? null : String(l2.ts),
|
|
||||||
slot: l2.slot == null ? null : String(l2.slot),
|
|
||||||
mid_price: numStr(mid),
|
|
||||||
best_bid_price: numStr(bestBid),
|
|
||||||
best_ask_price: numStr(bestAsk),
|
|
||||||
vwap_price: numStr(buy.vwap),
|
|
||||||
worst_price: numStr(buy.worstPrice),
|
|
||||||
filled_usd: numStr(buy.filledUsd),
|
|
||||||
filled_base: numStr(buy.filledBase),
|
|
||||||
impact_bps: numStr(buy.impactBps),
|
|
||||||
levels_consumed: buy.levelsConsumed,
|
|
||||||
fill_pct: numStr(buy.fillPct),
|
|
||||||
raw: {
|
|
||||||
ref: 'mid',
|
|
||||||
units: 'usd',
|
|
||||||
pricePrecision: cfg.pricePrecision,
|
|
||||||
basePrecision: cfg.basePrecision,
|
|
||||||
},
|
|
||||||
updated_at: updatedAt,
|
|
||||||
});
|
|
||||||
|
|
||||||
const sell = simulateSell(bids, mid, sizeUsd);
|
|
||||||
rows.push({
|
|
||||||
market_name: market,
|
|
||||||
side: 'sell',
|
|
||||||
size_usd: sizeUsd,
|
|
||||||
market_type: l2.market_type ? String(l2.market_type) : 'perp',
|
|
||||||
market_index: typeof l2.market_index === 'number' ? l2.market_index : null,
|
|
||||||
ts: l2.ts == null ? null : String(l2.ts),
|
|
||||||
slot: l2.slot == null ? null : String(l2.slot),
|
|
||||||
mid_price: numStr(mid),
|
|
||||||
best_bid_price: numStr(bestBid),
|
|
||||||
best_ask_price: numStr(bestAsk),
|
|
||||||
vwap_price: numStr(sell.vwap),
|
|
||||||
worst_price: numStr(sell.worstPrice),
|
|
||||||
filled_usd: numStr(sell.filledUsd),
|
|
||||||
filled_base: numStr(sell.filledBase),
|
|
||||||
impact_bps: numStr(sell.impactBps),
|
|
||||||
levels_consumed: sell.levelsConsumed,
|
|
||||||
fill_pct: numStr(sell.fillPct),
|
|
||||||
raw: {
|
|
||||||
ref: 'mid',
|
|
||||||
units: 'usd',
|
|
||||||
pricePrecision: cfg.pricePrecision,
|
|
||||||
basePrecision: cfg.basePrecision,
|
|
||||||
},
|
|
||||||
updated_at: updatedAt,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
) { affected_rows }
|
||||||
|
}
|
||||||
|
`;
|
||||||
|
await graphqlRequest(cfg, mutation, { rows: objectsV2 });
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(`[dlob-slippage-worker] fetch/compute: ${String(err?.message || err)}`);
|
console.error(`[dlob-slippage-worker] ${String(err?.message || err)}`);
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
await upsertSlippage(cfg, rows);
|
|
||||||
} catch (err) {
|
|
||||||
console.error(`[dlob-slippage-worker] upsert: ${String(err?.message || err)}`);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
await sleep(cfg.pollMs);
|
await sleep(cfg.pollMs);
|
||||||
|
|||||||
44
kustomize/base/dlob-ts-archiver/deployment-drift.yaml
Normal file
44
kustomize/base/dlob-ts-archiver/deployment-drift.yaml
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: dlob-ts-archiver-drift
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "6"
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: dlob-ts-archiver-drift
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: dlob-ts-archiver-drift
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: worker
|
||||||
|
image: node:20-slim
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
env:
|
||||||
|
- name: HASURA_GRAPHQL_URL
|
||||||
|
value: http://hasura:8080/v1/graphql
|
||||||
|
- name: HASURA_ADMIN_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: trade-hasura
|
||||||
|
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||||
|
- name: DLOB_SOURCE
|
||||||
|
value: drift
|
||||||
|
- name: DLOB_MARKETS
|
||||||
|
value: SOL-PERP,DOGE-PERP,JUP-PERP
|
||||||
|
- name: DLOB_TS_POLL_MS
|
||||||
|
value: "1000"
|
||||||
|
command: ["node", "/app/worker.mjs"]
|
||||||
|
volumeMounts:
|
||||||
|
- name: script
|
||||||
|
mountPath: /app/worker.mjs
|
||||||
|
subPath: worker.mjs
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: script
|
||||||
|
configMap:
|
||||||
|
name: dlob-ts-archiver-script
|
||||||
@@ -26,8 +26,10 @@ spec:
|
|||||||
secretKeyRef:
|
secretKeyRef:
|
||||||
name: trade-hasura
|
name: trade-hasura
|
||||||
key: HASURA_GRAPHQL_ADMIN_SECRET
|
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||||
|
- name: DLOB_SOURCE
|
||||||
|
value: mevnode
|
||||||
- name: DLOB_MARKETS
|
- name: DLOB_MARKETS
|
||||||
value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
|
value: SOL-PERP,DOGE-PERP,JUP-PERP
|
||||||
- name: DLOB_TS_POLL_MS
|
- name: DLOB_TS_POLL_MS
|
||||||
value: "1000"
|
value: "1000"
|
||||||
command: ["node", "/app/worker.mjs"]
|
command: ["node", "/app/worker.mjs"]
|
||||||
|
|||||||
@@ -1,6 +1,16 @@
|
|||||||
|
import fs from 'node:fs';
|
||||||
import process from 'node:process';
|
import process from 'node:process';
|
||||||
import { setTimeout as sleep } from 'node:timers/promises';
|
import { setTimeout as sleep } from 'node:timers/promises';
|
||||||
|
|
||||||
|
function readJsonFile(filePath) {
|
||||||
|
try {
|
||||||
|
const raw = fs.readFileSync(filePath, 'utf8');
|
||||||
|
return JSON.parse(raw);
|
||||||
|
} catch {
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
function getIsoNow() {
|
function getIsoNow() {
|
||||||
return new Date().toISOString();
|
return new Date().toISOString();
|
||||||
}
|
}
|
||||||
@@ -20,30 +30,54 @@ function envList(name, fallbackCsv) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
function resolveConfig() {
|
function resolveConfig() {
|
||||||
const hasuraUrl = String(process.env.HASURA_GRAPHQL_URL || 'http://hasura:8080/v1/graphql').trim();
|
const tokensPath =
|
||||||
const hasuraAdminSecret = String(process.env.HASURA_ADMIN_SECRET || '').trim();
|
process.env.HASURA_TOKENS_FILE ||
|
||||||
if (!hasuraAdminSecret) throw new Error('Missing HASURA_ADMIN_SECRET');
|
process.env.TOKENS_FILE ||
|
||||||
|
process.env.HASURA_CONFIG_FILE ||
|
||||||
|
'/app/tokens/hasura.json';
|
||||||
|
const tokens = readJsonFile(tokensPath) || {};
|
||||||
|
|
||||||
const markets = envList('DLOB_MARKETS', 'PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP');
|
const hasuraUrl =
|
||||||
const pollMs = clampInt(process.env.DLOB_TS_POLL_MS, 250, 60_000, 1000);
|
process.env.HASURA_GRAPHQL_URL ||
|
||||||
|
tokens.graphqlUrl ||
|
||||||
|
tokens.apiUrl ||
|
||||||
|
'http://hasura:8080/v1/graphql';
|
||||||
|
const hasuraAdminSecret =
|
||||||
|
process.env.HASURA_ADMIN_SECRET ||
|
||||||
|
process.env.HASURA_GRAPHQL_ADMIN_SECRET ||
|
||||||
|
tokens.adminSecret ||
|
||||||
|
tokens.hasuraAdminSecret;
|
||||||
|
const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined;
|
||||||
|
|
||||||
return { hasuraUrl, hasuraAdminSecret, markets, pollMs };
|
const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode';
|
||||||
|
const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
|
||||||
|
const pollMs = clampInt(process.env.DLOB_TS_POLL_MS, 500, 60_000, 1000);
|
||||||
|
|
||||||
|
return { hasuraUrl, hasuraAdminSecret, hasuraAuthToken, dlobSource, markets, pollMs };
|
||||||
}
|
}
|
||||||
|
|
||||||
async function graphqlRequest(cfg, query, variables) {
|
async function graphqlRequest(cfg, query, variables) {
|
||||||
|
const headers = { 'content-type': 'application/json' };
|
||||||
|
if (cfg.hasuraAuthToken) {
|
||||||
|
headers.authorization = `Bearer ${cfg.hasuraAuthToken}`;
|
||||||
|
} else if (cfg.hasuraAdminSecret) {
|
||||||
|
headers['x-hasura-admin-secret'] = cfg.hasuraAdminSecret;
|
||||||
|
} else {
|
||||||
|
throw new Error('Missing Hasura auth (set HASURA_AUTH_TOKEN or HASURA_ADMIN_SECRET or mount tokens/hasura.json)');
|
||||||
|
}
|
||||||
|
|
||||||
const res = await fetch(cfg.hasuraUrl, {
|
const res = await fetch(cfg.hasuraUrl, {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers,
|
||||||
'content-type': 'application/json',
|
|
||||||
'x-hasura-admin-secret': cfg.hasuraAdminSecret,
|
|
||||||
},
|
|
||||||
body: JSON.stringify({ query, variables }),
|
body: JSON.stringify({ query, variables }),
|
||||||
signal: AbortSignal.timeout(15_000),
|
signal: AbortSignal.timeout(15_000),
|
||||||
});
|
});
|
||||||
const text = await res.text();
|
const text = await res.text();
|
||||||
if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`);
|
if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`);
|
||||||
const json = JSON.parse(text);
|
const json = JSON.parse(text);
|
||||||
if (json.errors?.length) throw new Error(json.errors.map((e) => e.message).join(' | '));
|
if (json.errors?.length) {
|
||||||
|
throw new Error(json.errors.map((e) => e.message).join(' | '));
|
||||||
|
}
|
||||||
return json.data;
|
return json.data;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -63,6 +97,8 @@ async function main() {
|
|||||||
service: 'dlob-ts-archiver',
|
service: 'dlob-ts-archiver',
|
||||||
startedAt: getIsoNow(),
|
startedAt: getIsoNow(),
|
||||||
hasuraUrl: cfg.hasuraUrl,
|
hasuraUrl: cfg.hasuraUrl,
|
||||||
|
hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none',
|
||||||
|
dlobSource: cfg.dlobSource,
|
||||||
markets: cfg.markets,
|
markets: cfg.markets,
|
||||||
pollMs: cfg.pollMs,
|
pollMs: cfg.pollMs,
|
||||||
},
|
},
|
||||||
@@ -76,19 +112,24 @@ async function main() {
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
const query = `
|
const query = `
|
||||||
query Latest($markets: [String!]!) {
|
query Latest($source: String!, $markets: [String!]!) {
|
||||||
dlob_stats_latest(where: { market_name: { _in: $markets } }) {
|
dlob_stats_latest(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
|
||||||
market_name market_type market_index ts slot
|
market_name market_type market_index ts slot
|
||||||
mark_price oracle_price best_bid_price best_ask_price mid_price
|
mark_price oracle_price best_bid_price best_ask_price mid_price
|
||||||
spread_abs spread_bps depth_levels depth_bid_base depth_ask_base depth_bid_usd depth_ask_usd imbalance
|
spread_abs spread_bps depth_levels depth_bid_base depth_ask_base depth_bid_usd depth_ask_usd imbalance
|
||||||
raw
|
raw
|
||||||
}
|
}
|
||||||
dlob_depth_bps_latest(where: { market_name: { _in: $markets } }) {
|
dlob_depth_bps_latest(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
|
||||||
market_name band_bps market_type market_index ts slot
|
market_name band_bps market_type market_index ts slot
|
||||||
mid_price best_bid_price best_ask_price bid_base ask_base bid_usd ask_usd imbalance
|
mid_price best_bid_price best_ask_price bid_base ask_base bid_usd ask_usd imbalance
|
||||||
raw
|
raw
|
||||||
}
|
}
|
||||||
dlob_slippage_latest(where: { market_name: { _in: $markets } }) {
|
dlob_slippage_latest(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
|
||||||
|
market_name side size_usd market_type market_index ts slot
|
||||||
|
mid_price vwap_price worst_price filled_usd filled_base impact_bps levels_consumed fill_pct
|
||||||
|
raw
|
||||||
|
}
|
||||||
|
dlob_slippage_latest_v2(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
|
||||||
market_name side size_usd market_type market_index ts slot
|
market_name side size_usd market_type market_index ts slot
|
||||||
mid_price vwap_price worst_price filled_usd filled_base impact_bps levels_consumed fill_pct
|
mid_price vwap_price worst_price filled_usd filled_base impact_bps levels_consumed fill_pct
|
||||||
raw
|
raw
|
||||||
@@ -96,10 +137,11 @@ async function main() {
|
|||||||
}
|
}
|
||||||
`;
|
`;
|
||||||
|
|
||||||
const data = await graphqlRequest(cfg, query, { markets: cfg.markets });
|
const data = await graphqlRequest(cfg, query, { source: cfg.dlobSource, markets: cfg.markets });
|
||||||
|
|
||||||
const statsRows = (data?.dlob_stats_latest || []).map((r) => ({
|
const statsRows = (data?.dlob_stats_latest || []).map((r) => ({
|
||||||
ts: now,
|
ts: now,
|
||||||
|
source: cfg.dlobSource,
|
||||||
market_name: r.market_name,
|
market_name: r.market_name,
|
||||||
market_type: r.market_type,
|
market_type: r.market_type,
|
||||||
market_index: r.market_index ?? null,
|
market_index: r.market_index ?? null,
|
||||||
@@ -123,6 +165,7 @@ async function main() {
|
|||||||
|
|
||||||
const depthRows = (data?.dlob_depth_bps_latest || []).map((r) => ({
|
const depthRows = (data?.dlob_depth_bps_latest || []).map((r) => ({
|
||||||
ts: now,
|
ts: now,
|
||||||
|
source: cfg.dlobSource,
|
||||||
market_name: r.market_name,
|
market_name: r.market_name,
|
||||||
band_bps: r.band_bps,
|
band_bps: r.band_bps,
|
||||||
market_type: r.market_type,
|
market_type: r.market_type,
|
||||||
@@ -142,6 +185,7 @@ async function main() {
|
|||||||
|
|
||||||
const slippageRows = (data?.dlob_slippage_latest || []).map((r) => ({
|
const slippageRows = (data?.dlob_slippage_latest || []).map((r) => ({
|
||||||
ts: now,
|
ts: now,
|
||||||
|
source: cfg.dlobSource,
|
||||||
market_name: r.market_name,
|
market_name: r.market_name,
|
||||||
side: r.side,
|
side: r.side,
|
||||||
size_usd: r.size_usd,
|
size_usd: r.size_usd,
|
||||||
@@ -160,28 +204,42 @@ async function main() {
|
|||||||
raw: r.raw ?? null,
|
raw: r.raw ?? null,
|
||||||
}));
|
}));
|
||||||
|
|
||||||
if (!statsRows.length && !depthRows.length && !slippageRows.length) {
|
const slippageRowsV2 = (data?.dlob_slippage_latest_v2 || []).map((r) => ({
|
||||||
await sleep(cfg.pollMs);
|
ts: now,
|
||||||
continue;
|
source: cfg.dlobSource,
|
||||||
}
|
market_name: r.market_name,
|
||||||
|
side: r.side,
|
||||||
|
size_usd: r.size_usd,
|
||||||
|
market_type: r.market_type,
|
||||||
|
market_index: r.market_index ?? null,
|
||||||
|
source_ts: mapBigint(r.ts),
|
||||||
|
slot: mapBigint(r.slot),
|
||||||
|
mid_price: r.mid_price ?? null,
|
||||||
|
vwap_price: r.vwap_price ?? null,
|
||||||
|
worst_price: r.worst_price ?? null,
|
||||||
|
filled_usd: r.filled_usd ?? null,
|
||||||
|
filled_base: r.filled_base ?? null,
|
||||||
|
impact_bps: r.impact_bps ?? null,
|
||||||
|
levels_consumed: r.levels_consumed ?? null,
|
||||||
|
fill_pct: r.fill_pct ?? null,
|
||||||
|
raw: r.raw ?? null,
|
||||||
|
}));
|
||||||
|
|
||||||
const mutation = `
|
const mutation = `
|
||||||
mutation InsertTs(
|
mutation InsertTs(
|
||||||
$stats: [dlob_stats_ts_insert_input!]!
|
$stats: [dlob_stats_ts_insert_input!]!
|
||||||
$depth: [dlob_depth_bps_ts_insert_input!]!
|
$depth: [dlob_depth_bps_ts_insert_input!]!
|
||||||
$slip: [dlob_slippage_ts_insert_input!]!
|
$slip: [dlob_slippage_ts_insert_input!]!
|
||||||
|
$slipV2: [dlob_slippage_ts_v2_insert_input!]!
|
||||||
) {
|
) {
|
||||||
insert_dlob_stats_ts(objects: $stats) { affected_rows }
|
insert_dlob_stats_ts(objects: $stats) { affected_rows }
|
||||||
insert_dlob_depth_bps_ts(objects: $depth) { affected_rows }
|
insert_dlob_depth_bps_ts(objects: $depth) { affected_rows }
|
||||||
insert_dlob_slippage_ts(objects: $slip) { affected_rows }
|
insert_dlob_slippage_ts(objects: $slip) { affected_rows }
|
||||||
|
insert_dlob_slippage_ts_v2(objects: $slipV2) { affected_rows }
|
||||||
}
|
}
|
||||||
`;
|
`;
|
||||||
|
|
||||||
await graphqlRequest(cfg, mutation, {
|
await graphqlRequest(cfg, mutation, { stats: statsRows, depth: depthRows, slip: slippageRows, slipV2: slippageRowsV2 });
|
||||||
stats: statsRows,
|
|
||||||
depth: depthRows,
|
|
||||||
slip: slippageRows,
|
|
||||||
});
|
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
console.error(`[dlob-ts-archiver] ${String(err?.message || err)}`);
|
console.error(`[dlob-ts-archiver] ${String(err?.message || err)}`);
|
||||||
}
|
}
|
||||||
@@ -194,4 +252,3 @@ main().catch((err) => {
|
|||||||
console.error(String(err?.stack || err));
|
console.error(String(err?.stack || err));
|
||||||
process.exitCode = 1;
|
process.exitCode = 1;
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
52
kustomize/base/dlob-worker/deployment-drift.yaml
Normal file
52
kustomize/base/dlob-worker/deployment-drift.yaml
Normal file
@@ -0,0 +1,52 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: dlob-worker-drift
|
||||||
|
annotations:
|
||||||
|
argocd.argoproj.io/sync-wave: "5"
|
||||||
|
spec:
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app.kubernetes.io/name: dlob-worker-drift
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app.kubernetes.io/name: dlob-worker-drift
|
||||||
|
spec:
|
||||||
|
hostNetwork: true
|
||||||
|
dnsPolicy: ClusterFirstWithHostNet
|
||||||
|
containers:
|
||||||
|
- name: worker
|
||||||
|
image: node:20-slim
|
||||||
|
imagePullPolicy: IfNotPresent
|
||||||
|
env:
|
||||||
|
- name: HASURA_GRAPHQL_URL
|
||||||
|
value: http://hasura:8080/v1/graphql
|
||||||
|
- name: HASURA_ADMIN_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: trade-hasura
|
||||||
|
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||||
|
- name: DLOB_SOURCE
|
||||||
|
value: drift
|
||||||
|
- name: DLOB_HTTP_URL
|
||||||
|
value: https://dlob.drift.trade
|
||||||
|
- name: DLOB_FORCE_IPV6
|
||||||
|
value: "true"
|
||||||
|
- name: DLOB_MARKETS
|
||||||
|
value: SOL-PERP,DOGE-PERP,JUP-PERP
|
||||||
|
- name: DLOB_POLL_MS
|
||||||
|
value: "500"
|
||||||
|
- name: DLOB_DEPTH
|
||||||
|
value: "10"
|
||||||
|
command: ["node", "/app/worker.mjs"]
|
||||||
|
volumeMounts:
|
||||||
|
- name: script
|
||||||
|
mountPath: /app/worker.mjs
|
||||||
|
subPath: worker.mjs
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: script
|
||||||
|
configMap:
|
||||||
|
name: dlob-worker-script
|
||||||
@@ -26,10 +26,12 @@ spec:
|
|||||||
secretKeyRef:
|
secretKeyRef:
|
||||||
name: trade-hasura
|
name: trade-hasura
|
||||||
key: HASURA_GRAPHQL_ADMIN_SECRET
|
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||||
|
- name: DLOB_SOURCE
|
||||||
|
value: mevnode
|
||||||
- name: DLOB_HTTP_URL
|
- name: DLOB_HTTP_URL
|
||||||
value: http://dlob-server:6969
|
value: http://dlob-server:6969
|
||||||
- name: DLOB_MARKETS
|
- name: DLOB_MARKETS
|
||||||
value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
|
value: SOL-PERP,DOGE-PERP,JUP-PERP
|
||||||
- name: DLOB_POLL_MS
|
- name: DLOB_POLL_MS
|
||||||
value: "500"
|
value: "500"
|
||||||
- name: DLOB_DEPTH
|
- name: DLOB_DEPTH
|
||||||
|
|||||||
@@ -64,8 +64,9 @@ function resolveConfig() {
|
|||||||
.trim()
|
.trim()
|
||||||
.replace(/\/$/, '');
|
.replace(/\/$/, '');
|
||||||
const dlobForceIpv6 = envBool('DLOB_FORCE_IPV6', false);
|
const dlobForceIpv6 = envBool('DLOB_FORCE_IPV6', false);
|
||||||
|
const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode';
|
||||||
|
|
||||||
const markets = envList('DLOB_MARKETS', 'PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP');
|
const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
|
||||||
const depth = clampInt(process.env.DLOB_DEPTH, 1, 50, 10);
|
const depth = clampInt(process.env.DLOB_DEPTH, 1, 50, 10);
|
||||||
const pollMs = clampInt(process.env.DLOB_POLL_MS, 100, 10_000, 500);
|
const pollMs = clampInt(process.env.DLOB_POLL_MS, 100, 10_000, 500);
|
||||||
|
|
||||||
@@ -80,6 +81,7 @@ function resolveConfig() {
|
|||||||
hasuraUrl,
|
hasuraUrl,
|
||||||
hasuraAdminSecret,
|
hasuraAdminSecret,
|
||||||
hasuraAuthToken,
|
hasuraAuthToken,
|
||||||
|
dlobSource,
|
||||||
dlobHttpBase,
|
dlobHttpBase,
|
||||||
dlobForceIpv6,
|
dlobForceIpv6,
|
||||||
markets,
|
markets,
|
||||||
@@ -238,8 +240,9 @@ function computeStats({ l2, depth, pricePrecision, basePrecision }) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function l2ToInsertObject({ l2, updatedAt, pricePrecision }) {
|
function l2ToInsertObject({ dlobSource, l2, updatedAt, pricePrecision }) {
|
||||||
return {
|
return {
|
||||||
|
source: dlobSource,
|
||||||
market_name: String(l2.marketName),
|
market_name: String(l2.marketName),
|
||||||
market_type: String(l2.marketType || 'perp'),
|
market_type: String(l2.marketType || 'perp'),
|
||||||
market_index: typeof l2.marketIndex === 'number' ? l2.marketIndex : null,
|
market_index: typeof l2.marketIndex === 'number' ? l2.marketIndex : null,
|
||||||
@@ -256,8 +259,9 @@ function l2ToInsertObject({ l2, updatedAt, pricePrecision }) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function statsToInsertObject({ l2, stats, updatedAt }) {
|
function statsToInsertObject({ dlobSource, l2, stats, updatedAt }) {
|
||||||
return {
|
return {
|
||||||
|
source: dlobSource,
|
||||||
market_name: String(l2.marketName),
|
market_name: String(l2.marketName),
|
||||||
market_type: String(l2.marketType || 'perp'),
|
market_type: String(l2.marketType || 'perp'),
|
||||||
market_index: typeof l2.marketIndex === 'number' ? l2.marketIndex : null,
|
market_index: typeof l2.marketIndex === 'number' ? l2.marketIndex : null,
|
||||||
@@ -371,6 +375,7 @@ async function main() {
|
|||||||
startedAt: getIsoNow(),
|
startedAt: getIsoNow(),
|
||||||
hasuraUrl: cfg.hasuraUrl,
|
hasuraUrl: cfg.hasuraUrl,
|
||||||
hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none',
|
hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none',
|
||||||
|
dlobSource: cfg.dlobSource,
|
||||||
dlobHttpBase: cfg.dlobHttpBase,
|
dlobHttpBase: cfg.dlobHttpBase,
|
||||||
dlobForceIpv6: cfg.dlobForceIpv6,
|
dlobForceIpv6: cfg.dlobForceIpv6,
|
||||||
markets: cfg.markets,
|
markets: cfg.markets,
|
||||||
@@ -410,8 +415,8 @@ async function main() {
|
|||||||
basePrecision: cfg.basePrecision,
|
basePrecision: cfg.basePrecision,
|
||||||
});
|
});
|
||||||
|
|
||||||
l2Objects.push(l2ToInsertObject({ l2, updatedAt, pricePrecision: cfg.pricePrecision }));
|
l2Objects.push(l2ToInsertObject({ dlobSource: cfg.dlobSource, l2, updatedAt, pricePrecision: cfg.pricePrecision }));
|
||||||
statsObjects.push(statsToInsertObject({ l2, stats, updatedAt }));
|
statsObjects.push(statsToInsertObject({ dlobSource: cfg.dlobSource, l2, stats, updatedAt }));
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ spec:
|
|||||||
- name: REDIS_CLIENT
|
- name: REDIS_CLIENT
|
||||||
value: DLOB
|
value: DLOB
|
||||||
- name: PERP_MARKETS_TO_LOAD
|
- name: PERP_MARKETS_TO_LOAD
|
||||||
value: "0,1,2,4,75"
|
value: "0,7,24"
|
||||||
- name: ENDPOINT
|
- name: ENDPOINT
|
||||||
valueFrom:
|
valueFrom:
|
||||||
secretKeyRef:
|
secretKeyRef:
|
||||||
@@ -59,11 +59,15 @@ spec:
|
|||||||
httpGet:
|
httpGet:
|
||||||
path: /startup
|
path: /startup
|
||||||
port: http
|
port: http
|
||||||
initialDelaySeconds: 10
|
initialDelaySeconds: 120
|
||||||
periodSeconds: 10
|
periodSeconds: 10
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 30
|
||||||
livenessProbe:
|
livenessProbe:
|
||||||
httpGet:
|
httpGet:
|
||||||
path: /health
|
path: /health
|
||||||
port: http
|
port: http
|
||||||
initialDelaySeconds: 30
|
initialDelaySeconds: 240
|
||||||
periodSeconds: 20
|
periodSeconds: 20
|
||||||
|
timeoutSeconds: 3
|
||||||
|
failureThreshold: 10
|
||||||
|
|||||||
@@ -16,7 +16,7 @@ spec:
|
|||||||
- name: gitea-registry
|
- name: gitea-registry
|
||||||
containers:
|
containers:
|
||||||
- name: frontend
|
- name: frontend
|
||||||
image: gitea.mpabi.pl/trade/trade-frontend:sha-ob-20260111203413
|
image: gitea.mpabi.pl/trade/trade-frontend:sha-b06fe7f
|
||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
ports:
|
ports:
|
||||||
- name: http
|
- name: http
|
||||||
|
|||||||
@@ -89,6 +89,8 @@ async function main() {
|
|||||||
console.log(`[hasura-bootstrap] HASURA_URL=${HASURA_URL}`);
|
console.log(`[hasura-bootstrap] HASURA_URL=${HASURA_URL}`);
|
||||||
await waitForHasura();
|
await waitForHasura();
|
||||||
|
|
||||||
|
const PUBLIC_DLOB_SOURCE_HEADER = 'X-Hasura-Dlob-Source';
|
||||||
|
|
||||||
const apiTokensTable = { schema: 'public', name: 'api_tokens' };
|
const apiTokensTable = { schema: 'public', name: 'api_tokens' };
|
||||||
const source = 'default';
|
const source = 'default';
|
||||||
|
|
||||||
@@ -97,9 +99,12 @@ async function main() {
|
|||||||
const dlobStatsLatestTable = { schema: 'public', name: 'dlob_stats_latest' };
|
const dlobStatsLatestTable = { schema: 'public', name: 'dlob_stats_latest' };
|
||||||
const dlobDepthBpsLatestTable = { schema: 'public', name: 'dlob_depth_bps_latest' };
|
const dlobDepthBpsLatestTable = { schema: 'public', name: 'dlob_depth_bps_latest' };
|
||||||
const dlobSlippageLatestTable = { schema: 'public', name: 'dlob_slippage_latest' };
|
const dlobSlippageLatestTable = { schema: 'public', name: 'dlob_slippage_latest' };
|
||||||
|
const dlobSlippageLatestV2Table = { schema: 'public', name: 'dlob_slippage_latest_v2' };
|
||||||
|
const candlesCacheTable = { schema: 'public', name: 'drift_candles_cache' };
|
||||||
const dlobStatsTsTable = { schema: 'public', name: 'dlob_stats_ts' };
|
const dlobStatsTsTable = { schema: 'public', name: 'dlob_stats_ts' };
|
||||||
const dlobDepthBpsTsTable = { schema: 'public', name: 'dlob_depth_bps_ts' };
|
const dlobDepthBpsTsTable = { schema: 'public', name: 'dlob_depth_bps_ts' };
|
||||||
const dlobSlippageTsTable = { schema: 'public', name: 'dlob_slippage_ts' };
|
const dlobSlippageTsTable = { schema: 'public', name: 'dlob_slippage_ts' };
|
||||||
|
const dlobSlippageTsV2Table = { schema: 'public', name: 'dlob_slippage_ts_v2' };
|
||||||
const baseCandlesFn = { schema: 'public', name: 'get_drift_candles' };
|
const baseCandlesFn = { schema: 'public', name: 'get_drift_candles' };
|
||||||
const candlesReturnTable = { schema: 'public', name: 'drift_candles' };
|
const candlesReturnTable = { schema: 'public', name: 'drift_candles' };
|
||||||
|
|
||||||
@@ -169,7 +174,22 @@ async function main() {
|
|||||||
await ensureTickTable(t);
|
await ensureTickTable(t);
|
||||||
}
|
}
|
||||||
|
|
||||||
const ensureDlobTable = async (table, columns) => {
|
// Cached candles table (precomputed by worker; public read).
|
||||||
|
await ensurePublicSelectTable(candlesCacheTable, [
|
||||||
|
'bucket',
|
||||||
|
'bucket_seconds',
|
||||||
|
'symbol',
|
||||||
|
'source',
|
||||||
|
'open',
|
||||||
|
'high',
|
||||||
|
'low',
|
||||||
|
'close',
|
||||||
|
'oracle_close',
|
||||||
|
'ticks',
|
||||||
|
'updated_at',
|
||||||
|
]);
|
||||||
|
|
||||||
|
const ensureDlobTable = async (table, columns, { publicFilter } = {}) => {
|
||||||
await metadataIgnore({ type: 'pg_untrack_table', args: { source, table } });
|
await metadataIgnore({ type: 'pg_untrack_table', args: { source, table } });
|
||||||
await metadata({ type: 'pg_track_table', args: { source, table } });
|
await metadata({ type: 'pg_track_table', args: { source, table } });
|
||||||
|
|
||||||
@@ -182,7 +202,7 @@ async function main() {
|
|||||||
role: 'public',
|
role: 'public',
|
||||||
permission: {
|
permission: {
|
||||||
columns,
|
columns,
|
||||||
filter: {},
|
filter: publicFilter || {},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -218,7 +238,7 @@ async function main() {
|
|||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const ensurePublicSelectTable = async (table, columns) => {
|
async function ensurePublicSelectTable(table, columns, { publicFilter } = {}) {
|
||||||
await metadataIgnore({ type: 'pg_untrack_table', args: { source, table } });
|
await metadataIgnore({ type: 'pg_untrack_table', args: { source, table } });
|
||||||
await metadata({ type: 'pg_track_table', args: { source, table } });
|
await metadata({ type: 'pg_track_table', args: { source, table } });
|
||||||
|
|
||||||
@@ -231,7 +251,7 @@ async function main() {
|
|||||||
role: 'public',
|
role: 'public',
|
||||||
permission: {
|
permission: {
|
||||||
columns,
|
columns,
|
||||||
filter: {},
|
filter: publicFilter || {},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
@@ -239,9 +259,12 @@ async function main() {
|
|||||||
// Computed/archived tables are written by workers using admin secret; keep ingestor off by default.
|
// Computed/archived tables are written by workers using admin secret; keep ingestor off by default.
|
||||||
await metadataIgnore({ type: 'pg_drop_insert_permission', args: { source, table, role: 'ingestor' } });
|
await metadataIgnore({ type: 'pg_drop_insert_permission', args: { source, table, role: 'ingestor' } });
|
||||||
await metadataIgnore({ type: 'pg_drop_update_permission', args: { source, table, role: 'ingestor' } });
|
await metadataIgnore({ type: 'pg_drop_update_permission', args: { source, table, role: 'ingestor' } });
|
||||||
};
|
}
|
||||||
|
|
||||||
|
const dlobPublicFilter = { source: { _eq: PUBLIC_DLOB_SOURCE_HEADER } };
|
||||||
|
|
||||||
await ensureDlobTable(dlobL2LatestTable, [
|
await ensureDlobTable(dlobL2LatestTable, [
|
||||||
|
'source',
|
||||||
'market_name',
|
'market_name',
|
||||||
'market_type',
|
'market_type',
|
||||||
'market_index',
|
'market_index',
|
||||||
@@ -255,9 +278,10 @@ async function main() {
|
|||||||
'asks',
|
'asks',
|
||||||
'raw',
|
'raw',
|
||||||
'updated_at',
|
'updated_at',
|
||||||
]);
|
], { publicFilter: dlobPublicFilter });
|
||||||
|
|
||||||
await ensureDlobTable(dlobStatsLatestTable, [
|
await ensureDlobTable(dlobStatsLatestTable, [
|
||||||
|
'source',
|
||||||
'market_name',
|
'market_name',
|
||||||
'market_type',
|
'market_type',
|
||||||
'market_index',
|
'market_index',
|
||||||
@@ -278,9 +302,10 @@ async function main() {
|
|||||||
'imbalance',
|
'imbalance',
|
||||||
'raw',
|
'raw',
|
||||||
'updated_at',
|
'updated_at',
|
||||||
]);
|
], { publicFilter: dlobPublicFilter });
|
||||||
|
|
||||||
await ensurePublicSelectTable(dlobDepthBpsLatestTable, [
|
await ensurePublicSelectTable(dlobDepthBpsLatestTable, [
|
||||||
|
'source',
|
||||||
'market_name',
|
'market_name',
|
||||||
'band_bps',
|
'band_bps',
|
||||||
'market_type',
|
'market_type',
|
||||||
@@ -297,9 +322,10 @@ async function main() {
|
|||||||
'imbalance',
|
'imbalance',
|
||||||
'raw',
|
'raw',
|
||||||
'updated_at',
|
'updated_at',
|
||||||
]);
|
], { publicFilter: dlobPublicFilter });
|
||||||
|
|
||||||
await ensurePublicSelectTable(dlobSlippageLatestTable, [
|
await ensurePublicSelectTable(dlobSlippageLatestTable, [
|
||||||
|
'source',
|
||||||
'market_name',
|
'market_name',
|
||||||
'side',
|
'side',
|
||||||
'size_usd',
|
'size_usd',
|
||||||
@@ -319,11 +345,35 @@ async function main() {
|
|||||||
'fill_pct',
|
'fill_pct',
|
||||||
'raw',
|
'raw',
|
||||||
'updated_at',
|
'updated_at',
|
||||||
]);
|
], { publicFilter: dlobPublicFilter });
|
||||||
|
|
||||||
|
await ensurePublicSelectTable(dlobSlippageLatestV2Table, [
|
||||||
|
'source',
|
||||||
|
'market_name',
|
||||||
|
'side',
|
||||||
|
'size_usd',
|
||||||
|
'market_type',
|
||||||
|
'market_index',
|
||||||
|
'ts',
|
||||||
|
'slot',
|
||||||
|
'mid_price',
|
||||||
|
'best_bid_price',
|
||||||
|
'best_ask_price',
|
||||||
|
'vwap_price',
|
||||||
|
'worst_price',
|
||||||
|
'filled_usd',
|
||||||
|
'filled_base',
|
||||||
|
'impact_bps',
|
||||||
|
'levels_consumed',
|
||||||
|
'fill_pct',
|
||||||
|
'raw',
|
||||||
|
'updated_at',
|
||||||
|
], { publicFilter: dlobPublicFilter });
|
||||||
|
|
||||||
await ensurePublicSelectTable(dlobStatsTsTable, [
|
await ensurePublicSelectTable(dlobStatsTsTable, [
|
||||||
'ts',
|
'ts',
|
||||||
'id',
|
'id',
|
||||||
|
'source',
|
||||||
'market_name',
|
'market_name',
|
||||||
'market_type',
|
'market_type',
|
||||||
'market_index',
|
'market_index',
|
||||||
@@ -343,11 +393,12 @@ async function main() {
|
|||||||
'depth_ask_usd',
|
'depth_ask_usd',
|
||||||
'imbalance',
|
'imbalance',
|
||||||
'raw',
|
'raw',
|
||||||
]);
|
], { publicFilter: dlobPublicFilter });
|
||||||
|
|
||||||
await ensurePublicSelectTable(dlobDepthBpsTsTable, [
|
await ensurePublicSelectTable(dlobDepthBpsTsTable, [
|
||||||
'ts',
|
'ts',
|
||||||
'id',
|
'id',
|
||||||
|
'source',
|
||||||
'market_name',
|
'market_name',
|
||||||
'band_bps',
|
'band_bps',
|
||||||
'market_type',
|
'market_type',
|
||||||
@@ -363,11 +414,12 @@ async function main() {
|
|||||||
'ask_usd',
|
'ask_usd',
|
||||||
'imbalance',
|
'imbalance',
|
||||||
'raw',
|
'raw',
|
||||||
]);
|
], { publicFilter: dlobPublicFilter });
|
||||||
|
|
||||||
await ensurePublicSelectTable(dlobSlippageTsTable, [
|
await ensurePublicSelectTable(dlobSlippageTsTable, [
|
||||||
'ts',
|
'ts',
|
||||||
'id',
|
'id',
|
||||||
|
'source',
|
||||||
'market_name',
|
'market_name',
|
||||||
'side',
|
'side',
|
||||||
'size_usd',
|
'size_usd',
|
||||||
@@ -384,7 +436,29 @@ async function main() {
|
|||||||
'levels_consumed',
|
'levels_consumed',
|
||||||
'fill_pct',
|
'fill_pct',
|
||||||
'raw',
|
'raw',
|
||||||
]);
|
], { publicFilter: dlobPublicFilter });
|
||||||
|
|
||||||
|
await ensurePublicSelectTable(dlobSlippageTsV2Table, [
|
||||||
|
'ts',
|
||||||
|
'id',
|
||||||
|
'source',
|
||||||
|
'market_name',
|
||||||
|
'side',
|
||||||
|
'size_usd',
|
||||||
|
'market_type',
|
||||||
|
'market_index',
|
||||||
|
'source_ts',
|
||||||
|
'slot',
|
||||||
|
'mid_price',
|
||||||
|
'vwap_price',
|
||||||
|
'worst_price',
|
||||||
|
'filled_usd',
|
||||||
|
'filled_base',
|
||||||
|
'impact_bps',
|
||||||
|
'levels_consumed',
|
||||||
|
'fill_pct',
|
||||||
|
'raw',
|
||||||
|
], { publicFilter: dlobPublicFilter });
|
||||||
|
|
||||||
// Return table type for candle functions (needed for Hasura to track the function).
|
// Return table type for candle functions (needed for Hasura to track the function).
|
||||||
await metadataIgnore({ type: 'pg_track_table', args: { source, table: candlesReturnTable } });
|
await metadataIgnore({ type: 'pg_track_table', args: { source, table: candlesReturnTable } });
|
||||||
|
|||||||
@@ -121,10 +121,47 @@ CREATE TABLE IF NOT EXISTS public.drift_candles (
|
|||||||
high numeric,
|
high numeric,
|
||||||
low numeric,
|
low numeric,
|
||||||
close numeric,
|
close numeric,
|
||||||
|
oracle_open numeric,
|
||||||
|
oracle_high numeric,
|
||||||
|
oracle_low numeric,
|
||||||
oracle_close numeric,
|
oracle_close numeric,
|
||||||
ticks bigint
|
ticks bigint
|
||||||
);
|
);
|
||||||
|
|
||||||
|
ALTER TABLE public.drift_candles ADD COLUMN IF NOT EXISTS oracle_open numeric;
|
||||||
|
ALTER TABLE public.drift_candles ADD COLUMN IF NOT EXISTS oracle_high numeric;
|
||||||
|
ALTER TABLE public.drift_candles ADD COLUMN IF NOT EXISTS oracle_low numeric;
|
||||||
|
|
||||||
|
-- Precomputed candle cache (materialized by a worker).
|
||||||
|
-- Purpose: make tf switching instant by reading ready-made candles instead of aggregating `drift_ticks` on demand.
|
||||||
|
-- NOTE: `source=''` means "any source" (no source filter).
|
||||||
|
CREATE TABLE IF NOT EXISTS public.drift_candles_cache (
|
||||||
|
bucket timestamptz NOT NULL,
|
||||||
|
bucket_seconds integer NOT NULL,
|
||||||
|
symbol text NOT NULL,
|
||||||
|
source text NOT NULL DEFAULT '',
|
||||||
|
open numeric NOT NULL,
|
||||||
|
high numeric NOT NULL,
|
||||||
|
low numeric NOT NULL,
|
||||||
|
close numeric NOT NULL,
|
||||||
|
oracle_open numeric,
|
||||||
|
oracle_high numeric,
|
||||||
|
oracle_low numeric,
|
||||||
|
oracle_close numeric,
|
||||||
|
ticks bigint NOT NULL DEFAULT 0,
|
||||||
|
updated_at timestamptz NOT NULL DEFAULT now(),
|
||||||
|
PRIMARY KEY (bucket, bucket_seconds, symbol, source)
|
||||||
|
);
|
||||||
|
|
||||||
|
ALTER TABLE public.drift_candles_cache ADD COLUMN IF NOT EXISTS oracle_open numeric;
|
||||||
|
ALTER TABLE public.drift_candles_cache ADD COLUMN IF NOT EXISTS oracle_high numeric;
|
||||||
|
ALTER TABLE public.drift_candles_cache ADD COLUMN IF NOT EXISTS oracle_low numeric;
|
||||||
|
|
||||||
|
SELECT create_hypertable('drift_candles_cache', 'bucket', if_not_exists => TRUE, migrate_data => TRUE);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS drift_candles_cache_symbol_source_bucket_idx
|
||||||
|
ON public.drift_candles_cache (symbol, source, bucket_seconds, bucket DESC);
|
||||||
|
|
||||||
-- If an older version of the function exists with an incompatible return type,
|
-- If an older version of the function exists with an incompatible return type,
|
||||||
-- CREATE OR REPLACE will fail. Drop the old signature first (safe/idempotent).
|
-- CREATE OR REPLACE will fail. Drop the old signature first (safe/idempotent).
|
||||||
DROP FUNCTION IF EXISTS public.get_drift_candles(text, integer, integer, text);
|
DROP FUNCTION IF EXISTS public.get_drift_candles(text, integer, integer, text);
|
||||||
@@ -139,27 +176,144 @@ RETURNS SETOF public.drift_candles
|
|||||||
LANGUAGE sql
|
LANGUAGE sql
|
||||||
STABLE
|
STABLE
|
||||||
AS $$
|
AS $$
|
||||||
WITH base AS (
|
-- Zwraca zawsze "ciągłe" buckety (fill forward), nawet jeśli nie było ticków w danej sekundzie/minucie.
|
||||||
|
-- Dzięki temu frontend może rysować regularną oś czasu (np. 1px = 1s) bez dziwnych przeskoków.
|
||||||
|
WITH src AS (
|
||||||
|
SELECT COALESCE(p_source, '') AS source_key
|
||||||
|
),
|
||||||
|
raw_cached AS (
|
||||||
|
SELECT
|
||||||
|
c.bucket,
|
||||||
|
c.open,
|
||||||
|
c.high,
|
||||||
|
c.low,
|
||||||
|
c.close,
|
||||||
|
c.oracle_open,
|
||||||
|
c.oracle_high,
|
||||||
|
c.oracle_low,
|
||||||
|
c.oracle_close,
|
||||||
|
c.ticks
|
||||||
|
FROM public.drift_candles_cache c, src
|
||||||
|
WHERE c.symbol = p_symbol
|
||||||
|
AND c.bucket_seconds = p_bucket_seconds
|
||||||
|
AND c.source = src.source_key
|
||||||
|
ORDER BY c.bucket DESC
|
||||||
|
LIMIT p_limit
|
||||||
|
),
|
||||||
|
raw_fallback AS (
|
||||||
SELECT
|
SELECT
|
||||||
time_bucket(make_interval(secs => p_bucket_seconds), ts) AS bucket,
|
time_bucket(make_interval(secs => p_bucket_seconds), ts) AS bucket,
|
||||||
ts,
|
ts,
|
||||||
COALESCE(mark_price, oracle_price) AS px,
|
COALESCE(mark_price, oracle_price) AS px,
|
||||||
oracle_price AS oracle_px
|
oracle_price AS oracle_px
|
||||||
FROM public.drift_ticks
|
FROM public.drift_ticks, src
|
||||||
WHERE symbol = p_symbol
|
WHERE symbol = p_symbol
|
||||||
AND (p_source IS NULL OR source = p_source)
|
AND (src.source_key = '' OR source = src.source_key)
|
||||||
AND ts >= now() - make_interval(secs => (p_bucket_seconds * p_limit * 2))
|
AND ts >= now() - make_interval(secs => (p_bucket_seconds * p_limit * 2))
|
||||||
)
|
),
|
||||||
|
computed AS (
|
||||||
SELECT
|
SELECT
|
||||||
bucket,
|
bucket,
|
||||||
(array_agg(px ORDER BY ts ASC))[1] AS open,
|
(array_agg(px ORDER BY ts ASC))[1] AS open,
|
||||||
max(px) AS high,
|
max(px) AS high,
|
||||||
min(px) AS low,
|
min(px) AS low,
|
||||||
(array_agg(px ORDER BY ts DESC))[1] AS close,
|
(array_agg(px ORDER BY ts DESC))[1] AS close,
|
||||||
|
(array_agg(oracle_px ORDER BY ts ASC))[1] AS oracle_open,
|
||||||
|
max(oracle_px) AS oracle_high,
|
||||||
|
min(oracle_px) AS oracle_low,
|
||||||
(array_agg(oracle_px ORDER BY ts DESC))[1] AS oracle_close,
|
(array_agg(oracle_px ORDER BY ts DESC))[1] AS oracle_close,
|
||||||
count(*) AS ticks
|
count(*) AS ticks
|
||||||
FROM base
|
FROM raw_fallback
|
||||||
GROUP BY bucket
|
GROUP BY bucket
|
||||||
|
),
|
||||||
|
data AS (
|
||||||
|
SELECT * FROM raw_cached
|
||||||
|
UNION ALL
|
||||||
|
SELECT * FROM computed
|
||||||
|
WHERE NOT EXISTS (SELECT 1 FROM raw_cached)
|
||||||
|
),
|
||||||
|
bounds AS (
|
||||||
|
SELECT max(bucket) AS end_bucket FROM data
|
||||||
|
),
|
||||||
|
params AS (
|
||||||
|
SELECT
|
||||||
|
make_interval(secs => p_bucket_seconds) AS step,
|
||||||
|
make_interval(secs => (p_bucket_seconds * (p_limit - 1))) AS span
|
||||||
|
),
|
||||||
|
series AS (
|
||||||
|
SELECT generate_series(
|
||||||
|
bounds.end_bucket - params.span,
|
||||||
|
bounds.end_bucket,
|
||||||
|
params.step
|
||||||
|
) AS bucket
|
||||||
|
FROM bounds, params
|
||||||
|
WHERE bounds.end_bucket IS NOT NULL
|
||||||
|
),
|
||||||
|
joined AS (
|
||||||
|
SELECT
|
||||||
|
s.bucket,
|
||||||
|
d.open,
|
||||||
|
d.high,
|
||||||
|
d.low,
|
||||||
|
d.close,
|
||||||
|
d.oracle_open,
|
||||||
|
d.oracle_high,
|
||||||
|
d.oracle_low,
|
||||||
|
d.oracle_close,
|
||||||
|
d.ticks
|
||||||
|
FROM series s
|
||||||
|
LEFT JOIN data d USING (bucket)
|
||||||
|
ORDER BY s.bucket ASC
|
||||||
|
),
|
||||||
|
grouped AS (
|
||||||
|
SELECT
|
||||||
|
*,
|
||||||
|
sum(CASE WHEN close IS NOT NULL THEN 1 ELSE 0 END) OVER (ORDER BY bucket ASC) AS grp_close,
|
||||||
|
sum(CASE WHEN oracle_close IS NOT NULL THEN 1 ELSE 0 END) OVER (ORDER BY bucket ASC) AS grp_oracle
|
||||||
|
FROM joined
|
||||||
|
),
|
||||||
|
first_vals AS (
|
||||||
|
SELECT
|
||||||
|
(SELECT close FROM grouped WHERE close IS NOT NULL ORDER BY bucket ASC LIMIT 1) AS first_close,
|
||||||
|
(SELECT oracle_close FROM grouped WHERE oracle_close IS NOT NULL ORDER BY bucket ASC LIMIT 1) AS first_oracle
|
||||||
|
),
|
||||||
|
ff AS (
|
||||||
|
SELECT
|
||||||
|
g.bucket,
|
||||||
|
g.open,
|
||||||
|
g.high,
|
||||||
|
g.low,
|
||||||
|
g.close,
|
||||||
|
g.oracle_open,
|
||||||
|
g.oracle_high,
|
||||||
|
g.oracle_low,
|
||||||
|
g.oracle_close,
|
||||||
|
g.ticks,
|
||||||
|
COALESCE(
|
||||||
|
g.close,
|
||||||
|
max(g.close) OVER (PARTITION BY g.grp_close),
|
||||||
|
f.first_close
|
||||||
|
) AS ff_close,
|
||||||
|
COALESCE(
|
||||||
|
g.oracle_close,
|
||||||
|
max(g.oracle_close) OVER (PARTITION BY g.grp_oracle),
|
||||||
|
f.first_oracle
|
||||||
|
) AS ff_oracle
|
||||||
|
FROM grouped g
|
||||||
|
CROSS JOIN first_vals f
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
bucket,
|
||||||
|
COALESCE(open, ff_close) AS open,
|
||||||
|
COALESCE(high, ff_close) AS high,
|
||||||
|
COALESCE(low, ff_close) AS low,
|
||||||
|
COALESCE(close, ff_close) AS close,
|
||||||
|
COALESCE(oracle_open, ff_oracle) AS oracle_open,
|
||||||
|
COALESCE(oracle_high, ff_oracle) AS oracle_high,
|
||||||
|
COALESCE(oracle_low, ff_oracle) AS oracle_low,
|
||||||
|
COALESCE(oracle_close, ff_oracle) AS oracle_close,
|
||||||
|
COALESCE(ticks, 0) AS ticks
|
||||||
|
FROM ff
|
||||||
ORDER BY bucket DESC
|
ORDER BY bucket DESC
|
||||||
LIMIT p_limit;
|
LIMIT p_limit;
|
||||||
$$;
|
$$;
|
||||||
@@ -167,7 +321,8 @@ $$;
|
|||||||
-- Latest DLOB orderbook snapshots (top-N levels), per market.
|
-- Latest DLOB orderbook snapshots (top-N levels), per market.
|
||||||
-- Filled by a VPS worker (collector) and consumed by the UI via Hasura subscriptions.
|
-- Filled by a VPS worker (collector) and consumed by the UI via Hasura subscriptions.
|
||||||
CREATE TABLE IF NOT EXISTS public.dlob_l2_latest (
|
CREATE TABLE IF NOT EXISTS public.dlob_l2_latest (
|
||||||
market_name TEXT PRIMARY KEY,
|
source TEXT NOT NULL DEFAULT 'mevnode',
|
||||||
|
market_name TEXT NOT NULL,
|
||||||
market_type TEXT NOT NULL DEFAULT 'perp',
|
market_type TEXT NOT NULL DEFAULT 'perp',
|
||||||
market_index INTEGER,
|
market_index INTEGER,
|
||||||
ts BIGINT,
|
ts BIGINT,
|
||||||
@@ -179,15 +334,52 @@ CREATE TABLE IF NOT EXISTS public.dlob_l2_latest (
|
|||||||
bids JSONB,
|
bids JSONB,
|
||||||
asks JSONB,
|
asks JSONB,
|
||||||
raw JSONB,
|
raw JSONB,
|
||||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
PRIMARY KEY (source, market_name)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Schema upgrades (idempotent for existing volumes)
|
||||||
|
ALTER TABLE public.dlob_l2_latest ADD COLUMN IF NOT EXISTS source TEXT;
|
||||||
|
ALTER TABLE public.dlob_l2_latest ALTER COLUMN source SET DEFAULT 'mevnode';
|
||||||
|
UPDATE public.dlob_l2_latest SET source = 'mevnode' WHERE source IS NULL;
|
||||||
|
ALTER TABLE public.dlob_l2_latest ALTER COLUMN source SET NOT NULL;
|
||||||
|
|
||||||
|
-- Ensure PRIMARY KEY is (source, market_name) (required to keep 2 sources in parallel).
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
pk_name text;
|
||||||
|
pk_cols text[];
|
||||||
|
BEGIN
|
||||||
|
SELECT
|
||||||
|
con.conname,
|
||||||
|
array_agg(att.attname ORDER BY ord.ordinality)
|
||||||
|
INTO pk_name, pk_cols
|
||||||
|
FROM pg_constraint con
|
||||||
|
JOIN pg_class rel ON rel.oid = con.conrelid
|
||||||
|
JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
|
||||||
|
JOIN unnest(con.conkey) WITH ORDINALITY AS ord(attnum, ordinality) ON true
|
||||||
|
JOIN pg_attribute att ON att.attrelid = rel.oid AND att.attnum = ord.attnum
|
||||||
|
WHERE con.contype = 'p' AND nsp.nspname = 'public' AND rel.relname = 'dlob_l2_latest'
|
||||||
|
GROUP BY con.conname;
|
||||||
|
|
||||||
|
IF pk_name IS NULL THEN
|
||||||
|
EXECUTE 'ALTER TABLE public.dlob_l2_latest ADD CONSTRAINT dlob_l2_latest_pkey PRIMARY KEY (source, market_name)';
|
||||||
|
ELSIF pk_cols <> ARRAY['source','market_name'] THEN
|
||||||
|
EXECUTE format('ALTER TABLE public.dlob_l2_latest DROP CONSTRAINT %I', pk_name);
|
||||||
|
EXECUTE 'ALTER TABLE public.dlob_l2_latest ADD CONSTRAINT dlob_l2_latest_pkey PRIMARY KEY (source, market_name)';
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS dlob_l2_latest_updated_at_idx
|
CREATE INDEX IF NOT EXISTS dlob_l2_latest_updated_at_idx
|
||||||
ON public.dlob_l2_latest (updated_at DESC);
|
ON public.dlob_l2_latest (updated_at DESC);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_l2_latest_source_updated_at_idx
|
||||||
|
ON public.dlob_l2_latest (source, updated_at DESC);
|
||||||
|
|
||||||
-- Derived stats for fast UI display.
|
-- Derived stats for fast UI display.
|
||||||
CREATE TABLE IF NOT EXISTS public.dlob_stats_latest (
|
CREATE TABLE IF NOT EXISTS public.dlob_stats_latest (
|
||||||
market_name TEXT PRIMARY KEY,
|
source TEXT NOT NULL DEFAULT 'mevnode',
|
||||||
|
market_name TEXT NOT NULL,
|
||||||
market_type TEXT NOT NULL DEFAULT 'perp',
|
market_type TEXT NOT NULL DEFAULT 'perp',
|
||||||
market_index INTEGER,
|
market_index INTEGER,
|
||||||
ts BIGINT,
|
ts BIGINT,
|
||||||
@@ -206,15 +398,52 @@ CREATE TABLE IF NOT EXISTS public.dlob_stats_latest (
|
|||||||
depth_ask_usd NUMERIC,
|
depth_ask_usd NUMERIC,
|
||||||
imbalance NUMERIC,
|
imbalance NUMERIC,
|
||||||
raw JSONB,
|
raw JSONB,
|
||||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
PRIMARY KEY (source, market_name)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Schema upgrades (idempotent for existing volumes)
|
||||||
|
ALTER TABLE public.dlob_stats_latest ADD COLUMN IF NOT EXISTS source TEXT;
|
||||||
|
ALTER TABLE public.dlob_stats_latest ALTER COLUMN source SET DEFAULT 'mevnode';
|
||||||
|
UPDATE public.dlob_stats_latest SET source = 'mevnode' WHERE source IS NULL;
|
||||||
|
ALTER TABLE public.dlob_stats_latest ALTER COLUMN source SET NOT NULL;
|
||||||
|
|
||||||
|
-- Ensure PRIMARY KEY is (source, market_name) (required to keep 2 sources in parallel).
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
pk_name text;
|
||||||
|
pk_cols text[];
|
||||||
|
BEGIN
|
||||||
|
SELECT
|
||||||
|
con.conname,
|
||||||
|
array_agg(att.attname ORDER BY ord.ordinality)
|
||||||
|
INTO pk_name, pk_cols
|
||||||
|
FROM pg_constraint con
|
||||||
|
JOIN pg_class rel ON rel.oid = con.conrelid
|
||||||
|
JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
|
||||||
|
JOIN unnest(con.conkey) WITH ORDINALITY AS ord(attnum, ordinality) ON true
|
||||||
|
JOIN pg_attribute att ON att.attrelid = rel.oid AND att.attnum = ord.attnum
|
||||||
|
WHERE con.contype = 'p' AND nsp.nspname = 'public' AND rel.relname = 'dlob_stats_latest'
|
||||||
|
GROUP BY con.conname;
|
||||||
|
|
||||||
|
IF pk_name IS NULL THEN
|
||||||
|
EXECUTE 'ALTER TABLE public.dlob_stats_latest ADD CONSTRAINT dlob_stats_latest_pkey PRIMARY KEY (source, market_name)';
|
||||||
|
ELSIF pk_cols <> ARRAY['source','market_name'] THEN
|
||||||
|
EXECUTE format('ALTER TABLE public.dlob_stats_latest DROP CONSTRAINT %I', pk_name);
|
||||||
|
EXECUTE 'ALTER TABLE public.dlob_stats_latest ADD CONSTRAINT dlob_stats_latest_pkey PRIMARY KEY (source, market_name)';
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS dlob_stats_latest_updated_at_idx
|
CREATE INDEX IF NOT EXISTS dlob_stats_latest_updated_at_idx
|
||||||
ON public.dlob_stats_latest (updated_at DESC);
|
ON public.dlob_stats_latest (updated_at DESC);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_stats_latest_source_updated_at_idx
|
||||||
|
ON public.dlob_stats_latest (source, updated_at DESC);
|
||||||
|
|
||||||
-- Depth snapshots within bps bands around mid-price (per market, per band).
|
-- Depth snapshots within bps bands around mid-price (per market, per band).
|
||||||
-- Filled by a derived worker that reads `dlob_l2_latest`.
|
-- Filled by a derived worker that reads `dlob_l2_latest`.
|
||||||
CREATE TABLE IF NOT EXISTS public.dlob_depth_bps_latest (
|
CREATE TABLE IF NOT EXISTS public.dlob_depth_bps_latest (
|
||||||
|
source TEXT NOT NULL DEFAULT 'mevnode',
|
||||||
market_name TEXT NOT NULL,
|
market_name TEXT NOT NULL,
|
||||||
band_bps INTEGER NOT NULL,
|
band_bps INTEGER NOT NULL,
|
||||||
market_type TEXT NOT NULL DEFAULT 'perp',
|
market_type TEXT NOT NULL DEFAULT 'perp',
|
||||||
@@ -231,18 +460,54 @@ CREATE TABLE IF NOT EXISTS public.dlob_depth_bps_latest (
|
|||||||
imbalance NUMERIC,
|
imbalance NUMERIC,
|
||||||
raw JSONB,
|
raw JSONB,
|
||||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
PRIMARY KEY (market_name, band_bps)
|
PRIMARY KEY (source, market_name, band_bps)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Schema upgrades (idempotent for existing volumes)
|
||||||
|
ALTER TABLE public.dlob_depth_bps_latest ADD COLUMN IF NOT EXISTS source TEXT;
|
||||||
|
ALTER TABLE public.dlob_depth_bps_latest ALTER COLUMN source SET DEFAULT 'mevnode';
|
||||||
|
UPDATE public.dlob_depth_bps_latest SET source = 'mevnode' WHERE source IS NULL;
|
||||||
|
ALTER TABLE public.dlob_depth_bps_latest ALTER COLUMN source SET NOT NULL;
|
||||||
|
|
||||||
|
-- Ensure PRIMARY KEY is (source, market_name, band_bps) (required to keep 2 sources in parallel).
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
pk_name text;
|
||||||
|
pk_cols text[];
|
||||||
|
BEGIN
|
||||||
|
SELECT
|
||||||
|
con.conname,
|
||||||
|
array_agg(att.attname ORDER BY ord.ordinality)
|
||||||
|
INTO pk_name, pk_cols
|
||||||
|
FROM pg_constraint con
|
||||||
|
JOIN pg_class rel ON rel.oid = con.conrelid
|
||||||
|
JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
|
||||||
|
JOIN unnest(con.conkey) WITH ORDINALITY AS ord(attnum, ordinality) ON true
|
||||||
|
JOIN pg_attribute att ON att.attrelid = rel.oid AND att.attnum = ord.attnum
|
||||||
|
WHERE con.contype = 'p' AND nsp.nspname = 'public' AND rel.relname = 'dlob_depth_bps_latest'
|
||||||
|
GROUP BY con.conname;
|
||||||
|
|
||||||
|
IF pk_name IS NULL THEN
|
||||||
|
EXECUTE 'ALTER TABLE public.dlob_depth_bps_latest ADD CONSTRAINT dlob_depth_bps_latest_pkey PRIMARY KEY (source, market_name, band_bps)';
|
||||||
|
ELSIF pk_cols <> ARRAY['source','market_name','band_bps'] THEN
|
||||||
|
EXECUTE format('ALTER TABLE public.dlob_depth_bps_latest DROP CONSTRAINT %I', pk_name);
|
||||||
|
EXECUTE 'ALTER TABLE public.dlob_depth_bps_latest ADD CONSTRAINT dlob_depth_bps_latest_pkey PRIMARY KEY (source, market_name, band_bps)';
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS dlob_depth_bps_latest_updated_at_idx
|
CREATE INDEX IF NOT EXISTS dlob_depth_bps_latest_updated_at_idx
|
||||||
ON public.dlob_depth_bps_latest (updated_at DESC);
|
ON public.dlob_depth_bps_latest (updated_at DESC);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS dlob_depth_bps_latest_market_name_idx
|
CREATE INDEX IF NOT EXISTS dlob_depth_bps_latest_market_name_idx
|
||||||
ON public.dlob_depth_bps_latest (market_name);
|
ON public.dlob_depth_bps_latest (market_name);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_depth_bps_latest_source_market_name_idx
|
||||||
|
ON public.dlob_depth_bps_latest (source, market_name);
|
||||||
|
|
||||||
-- Slippage/impact estimates for "market" orders at common USD sizes.
|
-- Slippage/impact estimates for "market" orders at common USD sizes.
|
||||||
-- Filled by a derived worker that reads `dlob_l2_latest`.
|
-- Filled by a derived worker that reads `dlob_l2_latest`.
|
||||||
CREATE TABLE IF NOT EXISTS public.dlob_slippage_latest (
|
CREATE TABLE IF NOT EXISTS public.dlob_slippage_latest (
|
||||||
|
source TEXT NOT NULL DEFAULT 'mevnode',
|
||||||
market_name TEXT NOT NULL,
|
market_name TEXT NOT NULL,
|
||||||
side TEXT NOT NULL,
|
side TEXT NOT NULL,
|
||||||
size_usd INTEGER NOT NULL,
|
size_usd INTEGER NOT NULL,
|
||||||
@@ -262,22 +527,126 @@ CREATE TABLE IF NOT EXISTS public.dlob_slippage_latest (
|
|||||||
fill_pct NUMERIC,
|
fill_pct NUMERIC,
|
||||||
raw JSONB,
|
raw JSONB,
|
||||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
PRIMARY KEY (market_name, side, size_usd),
|
PRIMARY KEY (source, market_name, side, size_usd),
|
||||||
CONSTRAINT dlob_slippage_latest_side_chk CHECK (side IN ('buy', 'sell'))
|
CONSTRAINT dlob_slippage_latest_side_chk CHECK (side IN ('buy', 'sell'))
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Schema upgrades (idempotent for existing volumes)
|
||||||
|
ALTER TABLE public.dlob_slippage_latest ADD COLUMN IF NOT EXISTS source TEXT;
|
||||||
|
ALTER TABLE public.dlob_slippage_latest ALTER COLUMN source SET DEFAULT 'mevnode';
|
||||||
|
UPDATE public.dlob_slippage_latest SET source = 'mevnode' WHERE source IS NULL;
|
||||||
|
ALTER TABLE public.dlob_slippage_latest ALTER COLUMN source SET NOT NULL;
|
||||||
|
|
||||||
|
-- Ensure PRIMARY KEY is (source, market_name, side, size_usd) (required to keep 2 sources in parallel).
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
pk_name text;
|
||||||
|
pk_cols text[];
|
||||||
|
BEGIN
|
||||||
|
SELECT
|
||||||
|
con.conname,
|
||||||
|
array_agg(att.attname ORDER BY ord.ordinality)
|
||||||
|
INTO pk_name, pk_cols
|
||||||
|
FROM pg_constraint con
|
||||||
|
JOIN pg_class rel ON rel.oid = con.conrelid
|
||||||
|
JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
|
||||||
|
JOIN unnest(con.conkey) WITH ORDINALITY AS ord(attnum, ordinality) ON true
|
||||||
|
JOIN pg_attribute att ON att.attrelid = rel.oid AND att.attnum = ord.attnum
|
||||||
|
WHERE con.contype = 'p' AND nsp.nspname = 'public' AND rel.relname = 'dlob_slippage_latest'
|
||||||
|
GROUP BY con.conname;
|
||||||
|
|
||||||
|
IF pk_name IS NULL THEN
|
||||||
|
EXECUTE 'ALTER TABLE public.dlob_slippage_latest ADD CONSTRAINT dlob_slippage_latest_pkey PRIMARY KEY (source, market_name, side, size_usd)';
|
||||||
|
ELSIF pk_cols <> ARRAY['source','market_name','side','size_usd'] THEN
|
||||||
|
EXECUTE format('ALTER TABLE public.dlob_slippage_latest DROP CONSTRAINT %I', pk_name);
|
||||||
|
EXECUTE 'ALTER TABLE public.dlob_slippage_latest ADD CONSTRAINT dlob_slippage_latest_pkey PRIMARY KEY (source, market_name, side, size_usd)';
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_updated_at_idx
|
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_updated_at_idx
|
||||||
ON public.dlob_slippage_latest (updated_at DESC);
|
ON public.dlob_slippage_latest (updated_at DESC);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_market_name_idx
|
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_market_name_idx
|
||||||
ON public.dlob_slippage_latest (market_name);
|
ON public.dlob_slippage_latest (market_name);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_source_market_name_idx
|
||||||
|
ON public.dlob_slippage_latest (source, market_name);
|
||||||
|
|
||||||
|
-- Slippage v2: supports fractional order sizes (e.g. 0.1/0.2/0.5 USD), per market and side.
|
||||||
|
-- Keep v1 intact for backward compatibility and to avoid data loss.
|
||||||
|
CREATE TABLE IF NOT EXISTS public.dlob_slippage_latest_v2 (
|
||||||
|
source TEXT NOT NULL DEFAULT 'mevnode',
|
||||||
|
market_name TEXT NOT NULL,
|
||||||
|
side TEXT NOT NULL, -- buy|sell
|
||||||
|
size_usd NUMERIC NOT NULL,
|
||||||
|
market_type TEXT NOT NULL DEFAULT 'perp',
|
||||||
|
market_index INTEGER,
|
||||||
|
ts BIGINT,
|
||||||
|
slot BIGINT,
|
||||||
|
mid_price NUMERIC,
|
||||||
|
best_bid_price NUMERIC,
|
||||||
|
best_ask_price NUMERIC,
|
||||||
|
vwap_price NUMERIC,
|
||||||
|
worst_price NUMERIC,
|
||||||
|
filled_usd NUMERIC,
|
||||||
|
filled_base NUMERIC,
|
||||||
|
impact_bps NUMERIC,
|
||||||
|
levels_consumed INTEGER,
|
||||||
|
fill_pct NUMERIC,
|
||||||
|
raw JSONB,
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
PRIMARY KEY (source, market_name, side, size_usd),
|
||||||
|
CONSTRAINT dlob_slippage_latest_v2_side_chk CHECK (side IN ('buy', 'sell'))
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Schema upgrades (idempotent for existing volumes)
|
||||||
|
ALTER TABLE public.dlob_slippage_latest_v2 ADD COLUMN IF NOT EXISTS source TEXT;
|
||||||
|
ALTER TABLE public.dlob_slippage_latest_v2 ALTER COLUMN source SET DEFAULT 'mevnode';
|
||||||
|
UPDATE public.dlob_slippage_latest_v2 SET source = 'mevnode' WHERE source IS NULL;
|
||||||
|
ALTER TABLE public.dlob_slippage_latest_v2 ALTER COLUMN source SET NOT NULL;
|
||||||
|
|
||||||
|
-- Ensure PRIMARY KEY is (source, market_name, side, size_usd) (required to keep 2 sources in parallel).
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
pk_name text;
|
||||||
|
pk_cols text[];
|
||||||
|
BEGIN
|
||||||
|
SELECT
|
||||||
|
con.conname,
|
||||||
|
array_agg(att.attname ORDER BY ord.ordinality)
|
||||||
|
INTO pk_name, pk_cols
|
||||||
|
FROM pg_constraint con
|
||||||
|
JOIN pg_class rel ON rel.oid = con.conrelid
|
||||||
|
JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
|
||||||
|
JOIN unnest(con.conkey) WITH ORDINALITY AS ord(attnum, ordinality) ON true
|
||||||
|
JOIN pg_attribute att ON att.attrelid = rel.oid AND att.attnum = ord.attnum
|
||||||
|
WHERE con.contype = 'p' AND nsp.nspname = 'public' AND rel.relname = 'dlob_slippage_latest_v2'
|
||||||
|
GROUP BY con.conname;
|
||||||
|
|
||||||
|
IF pk_name IS NULL THEN
|
||||||
|
EXECUTE 'ALTER TABLE public.dlob_slippage_latest_v2 ADD CONSTRAINT dlob_slippage_latest_v2_pkey PRIMARY KEY (source, market_name, side, size_usd)';
|
||||||
|
ELSIF pk_cols <> ARRAY['source','market_name','side','size_usd'] THEN
|
||||||
|
EXECUTE format('ALTER TABLE public.dlob_slippage_latest_v2 DROP CONSTRAINT %I', pk_name);
|
||||||
|
EXECUTE 'ALTER TABLE public.dlob_slippage_latest_v2 ADD CONSTRAINT dlob_slippage_latest_v2_pkey PRIMARY KEY (source, market_name, side, size_usd)';
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_v2_updated_at_idx
|
||||||
|
ON public.dlob_slippage_latest_v2 (updated_at DESC);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_v2_market_name_idx
|
||||||
|
ON public.dlob_slippage_latest_v2 (market_name);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_v2_source_market_name_idx
|
||||||
|
ON public.dlob_slippage_latest_v2 (source, market_name);
|
||||||
|
|
||||||
-- Time-series tables for UI history (start: 7 days).
|
-- Time-series tables for UI history (start: 7 days).
|
||||||
-- Keep these append-only; use Timescale hypertables.
|
-- Keep these append-only; use Timescale hypertables.
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS public.dlob_stats_ts (
|
CREATE TABLE IF NOT EXISTS public.dlob_stats_ts (
|
||||||
ts TIMESTAMPTZ NOT NULL,
|
ts TIMESTAMPTZ NOT NULL,
|
||||||
id BIGSERIAL NOT NULL,
|
id BIGSERIAL NOT NULL,
|
||||||
|
source TEXT NOT NULL DEFAULT 'mevnode',
|
||||||
market_name TEXT NOT NULL,
|
market_name TEXT NOT NULL,
|
||||||
market_type TEXT NOT NULL DEFAULT 'perp',
|
market_type TEXT NOT NULL DEFAULT 'perp',
|
||||||
market_index INTEGER,
|
market_index INTEGER,
|
||||||
@@ -300,14 +669,24 @@ CREATE TABLE IF NOT EXISTS public.dlob_stats_ts (
|
|||||||
PRIMARY KEY (ts, id)
|
PRIMARY KEY (ts, id)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Schema upgrades (idempotent for existing volumes)
|
||||||
|
ALTER TABLE public.dlob_stats_ts ADD COLUMN IF NOT EXISTS source TEXT;
|
||||||
|
ALTER TABLE public.dlob_stats_ts ALTER COLUMN source SET DEFAULT 'mevnode';
|
||||||
|
UPDATE public.dlob_stats_ts SET source = 'mevnode' WHERE source IS NULL;
|
||||||
|
ALTER TABLE public.dlob_stats_ts ALTER COLUMN source SET NOT NULL;
|
||||||
|
|
||||||
SELECT create_hypertable('dlob_stats_ts', 'ts', if_not_exists => TRUE, migrate_data => TRUE);
|
SELECT create_hypertable('dlob_stats_ts', 'ts', if_not_exists => TRUE, migrate_data => TRUE);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS dlob_stats_ts_market_ts_desc_idx
|
CREATE INDEX IF NOT EXISTS dlob_stats_ts_market_ts_desc_idx
|
||||||
ON public.dlob_stats_ts (market_name, ts DESC);
|
ON public.dlob_stats_ts (market_name, ts DESC);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_stats_ts_source_market_ts_desc_idx
|
||||||
|
ON public.dlob_stats_ts (source, market_name, ts DESC);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS public.dlob_depth_bps_ts (
|
CREATE TABLE IF NOT EXISTS public.dlob_depth_bps_ts (
|
||||||
ts TIMESTAMPTZ NOT NULL,
|
ts TIMESTAMPTZ NOT NULL,
|
||||||
id BIGSERIAL NOT NULL,
|
id BIGSERIAL NOT NULL,
|
||||||
|
source TEXT NOT NULL DEFAULT 'mevnode',
|
||||||
market_name TEXT NOT NULL,
|
market_name TEXT NOT NULL,
|
||||||
band_bps INTEGER NOT NULL,
|
band_bps INTEGER NOT NULL,
|
||||||
market_type TEXT NOT NULL DEFAULT 'perp',
|
market_type TEXT NOT NULL DEFAULT 'perp',
|
||||||
@@ -326,14 +705,24 @@ CREATE TABLE IF NOT EXISTS public.dlob_depth_bps_ts (
|
|||||||
PRIMARY KEY (ts, id)
|
PRIMARY KEY (ts, id)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Schema upgrades (idempotent for existing volumes)
|
||||||
|
ALTER TABLE public.dlob_depth_bps_ts ADD COLUMN IF NOT EXISTS source TEXT;
|
||||||
|
ALTER TABLE public.dlob_depth_bps_ts ALTER COLUMN source SET DEFAULT 'mevnode';
|
||||||
|
UPDATE public.dlob_depth_bps_ts SET source = 'mevnode' WHERE source IS NULL;
|
||||||
|
ALTER TABLE public.dlob_depth_bps_ts ALTER COLUMN source SET NOT NULL;
|
||||||
|
|
||||||
SELECT create_hypertable('dlob_depth_bps_ts', 'ts', if_not_exists => TRUE, migrate_data => TRUE);
|
SELECT create_hypertable('dlob_depth_bps_ts', 'ts', if_not_exists => TRUE, migrate_data => TRUE);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS dlob_depth_bps_ts_market_ts_desc_idx
|
CREATE INDEX IF NOT EXISTS dlob_depth_bps_ts_market_ts_desc_idx
|
||||||
ON public.dlob_depth_bps_ts (market_name, ts DESC);
|
ON public.dlob_depth_bps_ts (market_name, ts DESC);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_depth_bps_ts_source_market_ts_desc_idx
|
||||||
|
ON public.dlob_depth_bps_ts (source, market_name, ts DESC);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS public.dlob_slippage_ts (
|
CREATE TABLE IF NOT EXISTS public.dlob_slippage_ts (
|
||||||
ts TIMESTAMPTZ NOT NULL,
|
ts TIMESTAMPTZ NOT NULL,
|
||||||
id BIGSERIAL NOT NULL,
|
id BIGSERIAL NOT NULL,
|
||||||
|
source TEXT NOT NULL DEFAULT 'mevnode',
|
||||||
market_name TEXT NOT NULL,
|
market_name TEXT NOT NULL,
|
||||||
side TEXT NOT NULL,
|
side TEXT NOT NULL,
|
||||||
size_usd INTEGER NOT NULL,
|
size_usd INTEGER NOT NULL,
|
||||||
@@ -353,11 +742,56 @@ CREATE TABLE IF NOT EXISTS public.dlob_slippage_ts (
|
|||||||
PRIMARY KEY (ts, id)
|
PRIMARY KEY (ts, id)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
-- Schema upgrades (idempotent for existing volumes)
|
||||||
|
ALTER TABLE public.dlob_slippage_ts ADD COLUMN IF NOT EXISTS source TEXT;
|
||||||
|
ALTER TABLE public.dlob_slippage_ts ALTER COLUMN source SET DEFAULT 'mevnode';
|
||||||
|
UPDATE public.dlob_slippage_ts SET source = 'mevnode' WHERE source IS NULL;
|
||||||
|
ALTER TABLE public.dlob_slippage_ts ALTER COLUMN source SET NOT NULL;
|
||||||
|
|
||||||
SELECT create_hypertable('dlob_slippage_ts', 'ts', if_not_exists => TRUE, migrate_data => TRUE);
|
SELECT create_hypertable('dlob_slippage_ts', 'ts', if_not_exists => TRUE, migrate_data => TRUE);
|
||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS dlob_slippage_ts_market_ts_desc_idx
|
CREATE INDEX IF NOT EXISTS dlob_slippage_ts_market_ts_desc_idx
|
||||||
ON public.dlob_slippage_ts (market_name, ts DESC);
|
ON public.dlob_slippage_ts (market_name, ts DESC);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_slippage_ts_source_market_ts_desc_idx
|
||||||
|
ON public.dlob_slippage_ts (source, market_name, ts DESC);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS public.dlob_slippage_ts_v2 (
|
||||||
|
ts TIMESTAMPTZ NOT NULL,
|
||||||
|
id BIGSERIAL NOT NULL,
|
||||||
|
source TEXT NOT NULL DEFAULT 'mevnode',
|
||||||
|
market_name TEXT NOT NULL,
|
||||||
|
side TEXT NOT NULL,
|
||||||
|
size_usd NUMERIC NOT NULL,
|
||||||
|
market_type TEXT NOT NULL DEFAULT 'perp',
|
||||||
|
market_index INTEGER,
|
||||||
|
source_ts BIGINT,
|
||||||
|
slot BIGINT,
|
||||||
|
mid_price NUMERIC,
|
||||||
|
vwap_price NUMERIC,
|
||||||
|
worst_price NUMERIC,
|
||||||
|
filled_usd NUMERIC,
|
||||||
|
filled_base NUMERIC,
|
||||||
|
impact_bps NUMERIC,
|
||||||
|
levels_consumed INTEGER,
|
||||||
|
fill_pct NUMERIC,
|
||||||
|
raw JSONB,
|
||||||
|
PRIMARY KEY (ts, id)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Schema upgrades (idempotent for existing volumes)
|
||||||
|
ALTER TABLE public.dlob_slippage_ts_v2 ADD COLUMN IF NOT EXISTS source TEXT;
|
||||||
|
ALTER TABLE public.dlob_slippage_ts_v2 ALTER COLUMN source SET DEFAULT 'mevnode';
|
||||||
|
UPDATE public.dlob_slippage_ts_v2 SET source = 'mevnode' WHERE source IS NULL;
|
||||||
|
ALTER TABLE public.dlob_slippage_ts_v2 ALTER COLUMN source SET NOT NULL;
|
||||||
|
|
||||||
|
SELECT create_hypertable('dlob_slippage_ts_v2', 'ts', if_not_exists => TRUE, migrate_data => TRUE);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_slippage_ts_v2_market_ts_desc_idx
|
||||||
|
ON public.dlob_slippage_ts_v2 (market_name, ts DESC);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_slippage_ts_v2_source_market_ts_desc_idx
|
||||||
|
ON public.dlob_slippage_ts_v2 (source, market_name, ts DESC);
|
||||||
-- Retention policies (best-effort; safe if Timescale is present).
|
-- Retention policies (best-effort; safe if Timescale is present).
|
||||||
DO $$
|
DO $$
|
||||||
BEGIN
|
BEGIN
|
||||||
@@ -375,3 +809,8 @@ BEGIN
|
|||||||
PERFORM add_retention_policy('dlob_slippage_ts', INTERVAL '7 days');
|
PERFORM add_retention_policy('dlob_slippage_ts', INTERVAL '7 days');
|
||||||
EXCEPTION WHEN OTHERS THEN
|
EXCEPTION WHEN OTHERS THEN
|
||||||
END $$;
|
END $$;
|
||||||
|
DO $$
|
||||||
|
BEGIN
|
||||||
|
PERFORM add_retention_policy('dlob_slippage_ts_v2', INTERVAL '7 days');
|
||||||
|
EXCEPTION WHEN OTHERS THEN
|
||||||
|
END $$;
|
||||||
|
|||||||
@@ -19,9 +19,14 @@ resources:
|
|||||||
- dlob/server-service.yaml
|
- dlob/server-service.yaml
|
||||||
- dlob/server-deployment.yaml
|
- dlob/server-deployment.yaml
|
||||||
- dlob-worker/deployment.yaml
|
- dlob-worker/deployment.yaml
|
||||||
|
- dlob-worker/deployment-drift.yaml
|
||||||
- dlob-depth-worker/deployment.yaml
|
- dlob-depth-worker/deployment.yaml
|
||||||
|
- dlob-depth-worker/deployment-drift.yaml
|
||||||
- dlob-slippage-worker/deployment.yaml
|
- dlob-slippage-worker/deployment.yaml
|
||||||
|
- dlob-slippage-worker/deployment-drift.yaml
|
||||||
- dlob-ts-archiver/deployment.yaml
|
- dlob-ts-archiver/deployment.yaml
|
||||||
|
- dlob-ts-archiver/deployment-drift.yaml
|
||||||
|
- candles-cache-worker/deployment.yaml
|
||||||
|
|
||||||
configMapGenerator:
|
configMapGenerator:
|
||||||
- name: postgres-initdb
|
- name: postgres-initdb
|
||||||
@@ -42,9 +47,15 @@ configMapGenerator:
|
|||||||
- name: dlob-ts-archiver-script
|
- name: dlob-ts-archiver-script
|
||||||
files:
|
files:
|
||||||
- dlob-ts-archiver/worker.mjs
|
- dlob-ts-archiver/worker.mjs
|
||||||
|
- name: candles-cache-worker-script
|
||||||
|
files:
|
||||||
|
- candles-cache-worker/worker.mjs
|
||||||
- name: trade-api-wrapper
|
- name: trade-api-wrapper
|
||||||
files:
|
files:
|
||||||
- api/wrapper.mjs
|
- api/wrapper.mjs
|
||||||
|
- name: trade-api-upstream
|
||||||
|
files:
|
||||||
|
- api/server.mjs
|
||||||
|
|
||||||
generatorOptions:
|
generatorOptions:
|
||||||
disableNameSuffixHash: true
|
disableNameSuffixHash: true
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
apiVersion: cert-manager.io/v1
|
||||||
|
kind: Certificate
|
||||||
|
metadata:
|
||||||
|
name: monitoring-mpabi-pl
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
secretName: monitoring-mpabi-pl-tls
|
||||||
|
issuerRef:
|
||||||
|
kind: ClusterIssuer
|
||||||
|
name: letsencrypt-prod
|
||||||
|
dnsNames:
|
||||||
|
- grafana.mpabi.pl
|
||||||
|
- prometheus.mpabi.pl
|
||||||
238
kustomize/infra/monitoring-extras/dashboard-agave-status.yaml
Normal file
238
kustomize/infra/monitoring-extras/dashboard-agave-status.yaml
Normal file
@@ -0,0 +1,238 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: grafana-dashboard-agave-status
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
grafana_dashboard: "1"
|
||||||
|
data:
|
||||||
|
agave-status.json: |-
|
||||||
|
{
|
||||||
|
"uid": "agave-status-mpabi",
|
||||||
|
"title": "Agave @ mpabi",
|
||||||
|
"timezone": "browser",
|
||||||
|
"schemaVersion": 39,
|
||||||
|
"version": 2,
|
||||||
|
"time": { "from": "now-6h", "to": "now" },
|
||||||
|
"timepicker": {
|
||||||
|
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m"],
|
||||||
|
"time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d"]
|
||||||
|
},
|
||||||
|
"refresh": "10s",
|
||||||
|
"tags": ["agave", "solana", "mpabi"],
|
||||||
|
"templating": {
|
||||||
|
"list": [
|
||||||
|
{
|
||||||
|
"name": "instance",
|
||||||
|
"type": "query",
|
||||||
|
"label": "Node Exporter Instance",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"query": { "query": "label_values(up{job=\"mpabi-node-exporter\"}, instance)", "refId": "PromVarInstance" },
|
||||||
|
"current": { "selected": false, "text": "10.66.66.1:9100", "value": "10.66.66.1:9100" }
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"panels": [
|
||||||
|
{
|
||||||
|
"id": 1,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "Geyser Metrics Target (Prometheus up)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "A", "expr": "up{job=\"mpabi-yellowstone-geyser\"}" }],
|
||||||
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "horizontal", "textMode": "value" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": { "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } },
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 2,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "agave-validator.service (systemd active)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "B", "expr": "node_systemd_unit_state{job=\"mpabi-node-exporter\",instance=\"$instance\",name=\"agave-validator.service\",state=\"active\"}" }],
|
||||||
|
"options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "horizontal", "textMode": "value" },
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": { "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } },
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 3,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "CPU Used (%)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "C", "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{job=\"mpabi-node-exporter\",instance=\"$instance\",mode=\"idle\"}[5m])) * 100)" }],
|
||||||
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
|
||||||
|
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 6 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 4,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Load (1m)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "D", "expr": "node_load1{job=\"mpabi-node-exporter\",instance=\"$instance\"}" }],
|
||||||
|
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 6 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 5,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Memory Used (%)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "E", "expr": "100 - (node_memory_MemAvailable_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\"} / node_memory_MemTotal_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\"} * 100)" }],
|
||||||
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
|
||||||
|
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 6 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 6,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Swap Used (GiB)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "F", "expr": "(node_memory_SwapTotal_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\"} - node_memory_SwapFree_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\"}) / 1024 / 1024 / 1024" }],
|
||||||
|
"fieldConfig": { "defaults": { "unit": "gbytes" }, "overrides": [] },
|
||||||
|
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 14 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 7,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Disk Free Accounts (%)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "G", "expr": "100 * node_filesystem_avail_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\",mountpoint=\"/var/lib/solana/accounts\"} / node_filesystem_size_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\",mountpoint=\"/var/lib/solana/accounts\"}" }],
|
||||||
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
|
||||||
|
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 14 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 8,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Disk Free Ledger (%)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "H", "expr": "100 * node_filesystem_avail_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\",mountpoint=\"/var/lib/solana/ledger\"} / node_filesystem_size_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\",mountpoint=\"/var/lib/solana/ledger\"}" }],
|
||||||
|
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
|
||||||
|
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 14 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 9,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Disk IO (NVMe) Read/Write (MiB/s)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [
|
||||||
|
{ "refId": "I", "expr": "sum by (device) (rate(node_disk_read_bytes_total{job=\"mpabi-node-exporter\",instance=\"$instance\",device=~\"nvme.*\"}[5m])) / 1024 / 1024", "legendFormat": "read {{device}}" },
|
||||||
|
{ "refId": "J", "expr": "sum by (device) (rate(node_disk_written_bytes_total{job=\"mpabi-node-exporter\",instance=\"$instance\",device=~\"nvme.*\"}[5m])) / 1024 / 1024", "legendFormat": "write {{device}}" }
|
||||||
|
],
|
||||||
|
"fieldConfig": { "defaults": { "unit": "mbytes" }, "overrides": [] },
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 10,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Network wg0 RX/TX (MiB/s)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [
|
||||||
|
{ "refId": "K", "expr": "rate(node_network_receive_bytes_total{job=\"mpabi-node-exporter\",instance=\"$instance\",device=\"wg0\"}[5m]) / 1024 / 1024", "legendFormat": "rx" },
|
||||||
|
{ "refId": "L", "expr": "rate(node_network_transmit_bytes_total{job=\"mpabi-node-exporter\",instance=\"$instance\",device=\"wg0\"}[5m]) / 1024 / 1024", "legendFormat": "tx" }
|
||||||
|
],
|
||||||
|
"fieldConfig": { "defaults": { "unit": "mbytes" }, "overrides": [] },
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 11,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Geyser: Subscriber Queue Size",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "M", "expr": "grpc_subscriber_queue_size{job=\"mpabi-yellowstone-geyser\"}", "legendFormat": "{{subscriber_id}}" }],
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 30 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 12,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Geyser: Connections Total",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "N", "expr": "connections_total{job=\"mpabi-yellowstone-geyser\"}" }],
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 30 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 13,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Geyser: Bytes Sent (MiB/s)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "O", "expr": "rate(grpc_bytes_sent{job=\"mpabi-yellowstone-geyser\"}[5m]) / 1024 / 1024", "legendFormat": "{{subscriber_id}}" }],
|
||||||
|
"fieldConfig": { "defaults": { "unit": "mbytes" }, "overrides": [] },
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 14,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Geyser: Messages Sent (/s)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "P", "expr": "rate(grpc_message_sent_count{job=\"mpabi-yellowstone-geyser\"}[5m])", "legendFormat": "{{subscriber_id}}" }],
|
||||||
|
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 38 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 15,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "Geyser: Disconnects (increase 15m)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "Q", "expr": "sum by (reason) (increase(grpc_client_disconnects_total{job=\"mpabi-yellowstone-geyser\"}[15m]))", "legendFormat": "{{reason}}" }],
|
||||||
|
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 46 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 16,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "RPC Slot Lag (slots)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "R", "expr": "solana_rpc_slot_lag{job=\"mpabi-node-exporter\",instance=\"$instance\"}" }],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "short",
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "yellow", "value": 20 },
|
||||||
|
{ "color": "red", "value": 50 }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 54 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 17,
|
||||||
|
"type": "stat",
|
||||||
|
"title": "RPC Slot Lag (szac. minuty)",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [{ "refId": "S", "expr": "solana_rpc_slot_lag{job=\"mpabi-node-exporter\",instance=\"$instance\"} * 0.4 / 60" }],
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "min",
|
||||||
|
"decimals": 2,
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{ "color": "green", "value": null },
|
||||||
|
{ "color": "yellow", "value": 1 },
|
||||||
|
{ "color": "red", "value": 2 }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 54 }
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 18,
|
||||||
|
"type": "timeseries",
|
||||||
|
"title": "RPC Slot & Reference Slot",
|
||||||
|
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||||
|
"targets": [
|
||||||
|
{ "refId": "T", "expr": "solana_rpc_slot{job=\"mpabi-node-exporter\",instance=\"$instance\"}" },
|
||||||
|
{ "refId": "U", "expr": "solana_rpc_slot_reference{job=\"mpabi-node-exporter\",instance=\"$instance\"}" },
|
||||||
|
{ "refId": "V", "expr": "solana_rpc_block_height{job=\"mpabi-node-exporter\",instance=\"$instance\"}" }
|
||||||
|
],
|
||||||
|
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 60 }
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: IngressRoute
|
||||||
|
metadata:
|
||||||
|
name: grafana-http
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
entryPoints:
|
||||||
|
- web
|
||||||
|
routes:
|
||||||
|
- match: Host(`grafana.mpabi.pl`)
|
||||||
|
kind: Rule
|
||||||
|
middlewares:
|
||||||
|
- name: redirect-to-https
|
||||||
|
services:
|
||||||
|
- name: monitoring-stack-grafana
|
||||||
|
port: 80
|
||||||
16
kustomize/infra/monitoring-extras/ingressroute-grafana.yaml
Normal file
16
kustomize/infra/monitoring-extras/ingressroute-grafana.yaml
Normal file
@@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: IngressRoute
|
||||||
|
metadata:
|
||||||
|
name: grafana
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
entryPoints:
|
||||||
|
- websecure
|
||||||
|
routes:
|
||||||
|
- match: Host(`grafana.mpabi.pl`)
|
||||||
|
kind: Rule
|
||||||
|
services:
|
||||||
|
- name: monitoring-stack-grafana
|
||||||
|
port: 80
|
||||||
|
tls:
|
||||||
|
secretName: monitoring-mpabi-pl-tls
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: IngressRoute
|
||||||
|
metadata:
|
||||||
|
name: prometheus-http
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
entryPoints:
|
||||||
|
- web
|
||||||
|
routes:
|
||||||
|
- match: Host(`prometheus.mpabi.pl`)
|
||||||
|
kind: Rule
|
||||||
|
middlewares:
|
||||||
|
- name: redirect-to-https
|
||||||
|
services:
|
||||||
|
- name: monitoring-stack-kube-prom-prometheus
|
||||||
|
port: 9090
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: IngressRoute
|
||||||
|
metadata:
|
||||||
|
name: prometheus
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
entryPoints:
|
||||||
|
- websecure
|
||||||
|
routes:
|
||||||
|
- match: Host(`prometheus.mpabi.pl`)
|
||||||
|
kind: Rule
|
||||||
|
services:
|
||||||
|
- name: monitoring-stack-kube-prom-prometheus
|
||||||
|
port: 9090
|
||||||
|
tls:
|
||||||
|
secretName: monitoring-mpabi-pl-tls
|
||||||
11
kustomize/infra/monitoring-extras/kustomization.yaml
Normal file
11
kustomize/infra/monitoring-extras/kustomization.yaml
Normal file
@@ -0,0 +1,11 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
resources:
|
||||||
|
- middleware-redirect-to-https.yaml
|
||||||
|
- certificate-monitoring-mpabi-pl.yaml
|
||||||
|
- ingressroute-grafana.yaml
|
||||||
|
- ingressroute-grafana-http.yaml
|
||||||
|
- ingressroute-prometheus.yaml
|
||||||
|
- ingressroute-prometheus-http.yaml
|
||||||
|
- dashboard-agave-status.yaml
|
||||||
|
- prometheus-rules-agave.yaml
|
||||||
@@ -0,0 +1,9 @@
|
|||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: Middleware
|
||||||
|
metadata:
|
||||||
|
name: redirect-to-https
|
||||||
|
namespace: monitoring
|
||||||
|
spec:
|
||||||
|
redirectScheme:
|
||||||
|
scheme: https
|
||||||
|
permanent: true
|
||||||
@@ -0,0 +1,61 @@
|
|||||||
|
apiVersion: monitoring.coreos.com/v1
|
||||||
|
kind: PrometheusRule
|
||||||
|
metadata:
|
||||||
|
name: agave-rpc-alerts
|
||||||
|
namespace: monitoring
|
||||||
|
labels:
|
||||||
|
app: kube-prometheus-stack
|
||||||
|
release: monitoring-stack
|
||||||
|
spec:
|
||||||
|
groups:
|
||||||
|
- name: agave-rpc-health
|
||||||
|
rules:
|
||||||
|
- alert: AgaveRPCDown
|
||||||
|
expr: "max by (instance) (solana_rpc_up{job=\"mpabi-node-exporter\"}) == 0"
|
||||||
|
for: 30s
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
team: mpabi
|
||||||
|
annotations:
|
||||||
|
summary: "Agave RPC is unreachable"
|
||||||
|
description: "RPC probe from node exporter reports solana_rpc_up == 0 for instance {{ $labels.instance }}."
|
||||||
|
- alert: AgaveRPCSlotLagHigh
|
||||||
|
expr: "sum by (instance) (solana_rpc_slot_lag{job=\"mpabi-node-exporter\"}) > 50"
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
team: mpabi
|
||||||
|
annotations:
|
||||||
|
summary: "Agave RPC is lagging behind cluster"
|
||||||
|
description: "Current slot lag is {{ $value }} for instance {{ $labels.instance }}. Reference endpoint in probe config may be misconfigured or validator is behind."
|
||||||
|
- alert: AgaveRPCSlotLagCritical
|
||||||
|
expr: "sum by (instance) (solana_rpc_slot_lag{job=\"mpabi-node-exporter\"}) > 500"
|
||||||
|
for: 2m
|
||||||
|
labels:
|
||||||
|
severity: critical
|
||||||
|
team: mpabi
|
||||||
|
annotations:
|
||||||
|
summary: "Agave RPC severe lag"
|
||||||
|
description: "Slot lag is critically high ({{ $value }} slots) on instance {{ $labels.instance }}."
|
||||||
|
- alert: AgaveIOHigh
|
||||||
|
expr: |
|
||||||
|
sum by (instance) (
|
||||||
|
(rate(node_disk_read_bytes_total{job="mpabi-node-exporter",device=~"nvme.*"}[5m]) +
|
||||||
|
rate(node_disk_written_bytes_total{job="mpabi-node-exporter",device=~"nvme.*"}[5m])) / 1024 / 1024
|
||||||
|
) > 300
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
team: mpabi
|
||||||
|
annotations:
|
||||||
|
summary: "High storage I/O on Agave node"
|
||||||
|
description: "Combined NVMe read+write throughput >300 MiB/s for 5m on {{ $labels.instance }}. Check disk pressure and Geyser/ledger workload."
|
||||||
|
- alert: AgaveIOWaitHigh
|
||||||
|
expr: "avg by (instance) (rate(node_cpu_seconds_total{job=\"mpabi-node-exporter\",mode=\"iowait\"}[5m])) > 0.2"
|
||||||
|
for: 5m
|
||||||
|
labels:
|
||||||
|
severity: warning
|
||||||
|
team: mpabi
|
||||||
|
annotations:
|
||||||
|
summary: "High iowait on Agave node"
|
||||||
|
description: "Iowait over 20% on average for 5m on {{ $labels.instance }}. Storage latency is likely impacting slot progress."
|
||||||
14
kustomize/overlays/prod/dlob-rpc-endpoint-patch.yaml
Normal file
14
kustomize/overlays/prod/dlob-rpc-endpoint-patch.yaml
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: dlob-publisher
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: publisher
|
||||||
|
env:
|
||||||
|
- name: ENDPOINT
|
||||||
|
value: "http://10.66.66.1:8899"
|
||||||
|
- name: WS_ENDPOINT
|
||||||
|
value: "ws://10.66.66.1:8900"
|
||||||
14
kustomize/overlays/prod/dlob-rpc-server-endpoint-patch.yaml
Normal file
14
kustomize/overlays/prod/dlob-rpc-server-endpoint-patch.yaml
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: dlob-server
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: server
|
||||||
|
env:
|
||||||
|
- name: ENDPOINT
|
||||||
|
value: "http://10.66.66.1:8899"
|
||||||
|
- name: WS_ENDPOINT
|
||||||
|
value: "ws://10.66.66.1:8900"
|
||||||
18
kustomize/overlays/prod/frontend-graphql-proxy-patch.yaml
Normal file
18
kustomize/overlays/prod/frontend-graphql-proxy-patch.yaml
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: trade-frontend
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: frontend
|
||||||
|
volumeMounts:
|
||||||
|
- name: frontend-server-script
|
||||||
|
mountPath: /app/services/frontend/server.mjs
|
||||||
|
subPath: frontend-server.mjs
|
||||||
|
readOnly: true
|
||||||
|
volumes:
|
||||||
|
- name: frontend-server-script
|
||||||
|
configMap:
|
||||||
|
name: trade-frontend-server-script
|
||||||
813
kustomize/overlays/prod/frontend-server.mjs
Normal file
813
kustomize/overlays/prod/frontend-server.mjs
Normal file
@@ -0,0 +1,813 @@
|
|||||||
|
import crypto from 'node:crypto';
|
||||||
|
import { spawnSync } from 'node:child_process';
|
||||||
|
import fs from 'node:fs';
|
||||||
|
import http from 'node:http';
|
||||||
|
import https from 'node:https';
|
||||||
|
import net from 'node:net';
|
||||||
|
import path from 'node:path';
|
||||||
|
import tls from 'node:tls';
|
||||||
|
|
||||||
|
const PORT = Number.parseInt(process.env.PORT || '8081', 10);
|
||||||
|
if (!Number.isInteger(PORT) || PORT <= 0) throw new Error(`Invalid PORT: ${process.env.PORT}`);
|
||||||
|
|
||||||
|
const APP_VERSION = String(process.env.APP_VERSION || 'v1').trim() || 'v1';
|
||||||
|
const BUILD_TIMESTAMP = String(process.env.BUILD_TIMESTAMP || '').trim() || undefined;
|
||||||
|
const STARTED_AT = new Date().toISOString();
|
||||||
|
|
||||||
|
const STATIC_DIR = process.env.STATIC_DIR || '/srv';
|
||||||
|
const BASIC_AUTH_FILE = process.env.BASIC_AUTH_FILE || '/tokens/frontend.json';
|
||||||
|
const API_READ_TOKEN_FILE = process.env.API_READ_TOKEN_FILE || '/tokens/read.json';
|
||||||
|
const API_UPSTREAM = process.env.API_UPSTREAM || process.env.API_URL || 'http://api:8787';
|
||||||
|
const HASURA_UPSTREAM = process.env.HASURA_UPSTREAM || 'http://hasura:8080';
|
||||||
|
const HASURA_GRAPHQL_PATH = process.env.HASURA_GRAPHQL_PATH || '/v1/graphql';
|
||||||
|
const GRAPHQL_CORS_ORIGIN = process.env.GRAPHQL_CORS_ORIGIN || process.env.CORS_ORIGIN || '*';
|
||||||
|
const BASIC_AUTH_MODE = String(process.env.BASIC_AUTH_MODE || 'on')
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
const BASIC_AUTH_ENABLED = !['off', 'false', '0', 'disabled', 'none'].includes(BASIC_AUTH_MODE);
|
||||||
|
const AUTH_USER_HEADER = String(process.env.AUTH_USER_HEADER || 'x-trade-user')
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
const AUTH_MODE = String(process.env.AUTH_MODE || 'session')
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
const HTPASSWD_FILE = String(process.env.HTPASSWD_FILE || '/auth/users').trim();
|
||||||
|
const AUTH_SESSION_SECRET_FILE = String(process.env.AUTH_SESSION_SECRET_FILE || '').trim() || null;
|
||||||
|
const AUTH_SESSION_COOKIE = String(process.env.AUTH_SESSION_COOKIE || 'trade_session')
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
const AUTH_SESSION_TTL_SECONDS = Number.parseInt(process.env.AUTH_SESSION_TTL_SECONDS || '43200', 10); // 12h
|
||||||
|
const DLOB_SOURCE_COOKIE = String(process.env.DLOB_SOURCE_COOKIE || 'trade_dlob_source').trim() || 'trade_dlob_source';
|
||||||
|
const DLOB_SOURCE_DEFAULT = String(process.env.DLOB_SOURCE_DEFAULT || 'mevnode').trim() || 'mevnode';
|
||||||
|
|
||||||
|
function readJson(filePath) {
|
||||||
|
const raw = fs.readFileSync(filePath, 'utf8');
|
||||||
|
return JSON.parse(raw);
|
||||||
|
}
|
||||||
|
|
||||||
|
function readText(filePath) {
|
||||||
|
return fs.readFileSync(filePath, 'utf8');
|
||||||
|
}
|
||||||
|
|
||||||
|
function timingSafeEqualStr(a, b) {
|
||||||
|
const aa = Buffer.from(String(a), 'utf8');
|
||||||
|
const bb = Buffer.from(String(b), 'utf8');
|
||||||
|
if (aa.length !== bb.length) return false;
|
||||||
|
return crypto.timingSafeEqual(aa, bb);
|
||||||
|
}
|
||||||
|
|
||||||
|
function timingSafeEqualBuf(a, b) {
|
||||||
|
if (!(a instanceof Uint8Array) || !(b instanceof Uint8Array)) return false;
|
||||||
|
if (a.length !== b.length) return false;
|
||||||
|
return crypto.timingSafeEqual(Buffer.from(a), Buffer.from(b));
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadBasicAuth() {
|
||||||
|
const j = readJson(BASIC_AUTH_FILE);
|
||||||
|
const username = (j?.username || '').toString();
|
||||||
|
const password = (j?.password || '').toString();
|
||||||
|
if (!username || !password) throw new Error(`Invalid BASIC_AUTH_FILE: ${BASIC_AUTH_FILE}`);
|
||||||
|
return { username, password };
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadApiReadToken() {
|
||||||
|
const j = readJson(API_READ_TOKEN_FILE);
|
||||||
|
const token = (j?.token || '').toString();
|
||||||
|
if (!token) throw new Error(`Invalid API_READ_TOKEN_FILE: ${API_READ_TOKEN_FILE}`);
|
||||||
|
return token;
|
||||||
|
}
|
||||||
|
|
||||||
|
function send(res, status, headers, body) {
|
||||||
|
res.statusCode = status;
|
||||||
|
for (const [k, v] of Object.entries(headers || {})) res.setHeader(k, v);
|
||||||
|
if (body == null) return void res.end();
|
||||||
|
res.end(body);
|
||||||
|
}
|
||||||
|
|
||||||
|
function sendJson(res, status, body) {
|
||||||
|
send(res, status, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' }, JSON.stringify(body));
|
||||||
|
}
|
||||||
|
|
||||||
|
function basicAuthRequired(res) {
|
||||||
|
res.setHeader('www-authenticate', 'Basic realm="trade"');
|
||||||
|
send(res, 401, { 'content-type': 'text/plain; charset=utf-8' }, 'unauthorized');
|
||||||
|
}
|
||||||
|
|
||||||
|
function unauthorized(res) {
|
||||||
|
sendJson(res, 401, { ok: false, error: 'unauthorized' });
|
||||||
|
}
|
||||||
|
|
||||||
|
function isAuthorized(req, creds) {
|
||||||
|
const auth = req.headers.authorization || '';
|
||||||
|
const m = String(auth).match(/^Basic\s+(.+)$/i);
|
||||||
|
if (!m?.[1]) return false;
|
||||||
|
let decoded;
|
||||||
|
try {
|
||||||
|
decoded = Buffer.from(m[1], 'base64').toString('utf8');
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
const idx = decoded.indexOf(':');
|
||||||
|
if (idx < 0) return false;
|
||||||
|
const u = decoded.slice(0, idx);
|
||||||
|
const p = decoded.slice(idx + 1);
|
||||||
|
return timingSafeEqualStr(u, creds.username) && timingSafeEqualStr(p, creds.password);
|
||||||
|
}
|
||||||
|
|
||||||
|
const MIME = {
|
||||||
|
'.html': 'text/html; charset=utf-8',
|
||||||
|
'.css': 'text/css; charset=utf-8',
|
||||||
|
'.js': 'application/javascript; charset=utf-8',
|
||||||
|
'.mjs': 'application/javascript; charset=utf-8',
|
||||||
|
'.json': 'application/json; charset=utf-8',
|
||||||
|
'.svg': 'image/svg+xml',
|
||||||
|
'.png': 'image/png',
|
||||||
|
'.jpg': 'image/jpeg',
|
||||||
|
'.jpeg': 'image/jpeg',
|
||||||
|
'.gif': 'image/gif',
|
||||||
|
'.ico': 'image/x-icon',
|
||||||
|
'.txt': 'text/plain; charset=utf-8',
|
||||||
|
'.map': 'application/json; charset=utf-8',
|
||||||
|
};
|
||||||
|
|
||||||
|
function contentTypeFor(filePath) {
|
||||||
|
return MIME[path.extname(filePath).toLowerCase()] || 'application/octet-stream';
|
||||||
|
}
|
||||||
|
|
||||||
|
function safePathFromUrlPath(urlPath) {
|
||||||
|
const decoded = decodeURIComponent(urlPath);
|
||||||
|
const cleaned = decoded.replace(/\0/g, '');
|
||||||
|
// strip leading slash so join() doesn't ignore STATIC_DIR
|
||||||
|
const rel = cleaned.replace(/^\/+/, '');
|
||||||
|
const normalized = path.normalize(rel);
|
||||||
|
// prevent traversal
|
||||||
|
if (normalized.startsWith('..') || path.isAbsolute(normalized)) return null;
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
function injectIndexHtml(html, { dlobSource, redirectPath }) {
|
||||||
|
const src = normalizeDlobSource(dlobSource) || 'mevnode';
|
||||||
|
const redirect = safeRedirectPath(redirectPath);
|
||||||
|
const hrefBase = `/prefs/dlob-source?redirect=${encodeURIComponent(redirect)}&set=`;
|
||||||
|
|
||||||
|
const styleActive = 'font-weight:700;text-decoration:underline;';
|
||||||
|
const styleInactive = 'font-weight:400;text-decoration:none;';
|
||||||
|
|
||||||
|
const snippet = `
|
||||||
|
<!-- trade: dlob source switch -->
|
||||||
|
<div style="position:fixed;right:12px;bottom:12px;z-index:2147483647;background:rgba(0,0,0,0.72);color:#fff;padding:8px 10px;border-radius:10px;font:12px/1.2 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;backdrop-filter:blur(6px);">
|
||||||
|
<span style="opacity:0.85;margin-right:6px;">DLOB</span>
|
||||||
|
<a href="${hrefBase}mevnode" style="color:#fff;${src === 'mevnode' ? styleActive : styleInactive}">mevnode</a>
|
||||||
|
<span style="opacity:0.6;margin:0 6px;">|</span>
|
||||||
|
<a href="${hrefBase}drift" style="color:#fff;${src === 'drift' ? styleActive : styleInactive}">drift</a>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
const bodyClose = /<\/body>/i;
|
||||||
|
if (bodyClose.test(html)) return html.replace(bodyClose, `${snippet}</body>`);
|
||||||
|
return `${html}\n${snippet}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function serveStatic(req, res) {
|
||||||
|
if (req.method !== 'GET' && req.method !== 'HEAD') {
|
||||||
|
send(res, 405, { 'content-type': 'text/plain; charset=utf-8' }, 'method_not_allowed');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||||
|
const rel = safePathFromUrlPath(url.pathname);
|
||||||
|
if (rel == null) {
|
||||||
|
send(res, 400, { 'content-type': 'text/plain; charset=utf-8' }, 'bad_request');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const root = path.resolve(STATIC_DIR);
|
||||||
|
const fileCandidate = path.resolve(root, rel);
|
||||||
|
if (!fileCandidate.startsWith(root)) {
|
||||||
|
send(res, 400, { 'content-type': 'text/plain; charset=utf-8' }, 'bad_request');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const trySend = (filePath) => {
|
||||||
|
try {
|
||||||
|
const st = fs.statSync(filePath);
|
||||||
|
if (st.isDirectory()) return trySend(path.join(filePath, 'index.html'));
|
||||||
|
res.statusCode = 200;
|
||||||
|
res.setHeader('content-type', contentTypeFor(filePath));
|
||||||
|
res.setHeader('cache-control', filePath.endsWith('index.html') ? 'no-cache' : 'public, max-age=31536000');
|
||||||
|
if (req.method === 'HEAD') return void res.end();
|
||||||
|
if (filePath.endsWith('index.html')) {
|
||||||
|
const html = fs.readFileSync(filePath, 'utf8');
|
||||||
|
const injected = injectIndexHtml(html, {
|
||||||
|
dlobSource: resolveDlobSource(req),
|
||||||
|
redirectPath: url.pathname + url.search,
|
||||||
|
});
|
||||||
|
res.end(injected);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
fs.createReadStream(filePath).pipe(res);
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// exact file, otherwise SPA fallback
|
||||||
|
if (trySend(fileCandidate)) return;
|
||||||
|
const indexPath = path.join(root, 'index.html');
|
||||||
|
if (trySend(indexPath)) return;
|
||||||
|
|
||||||
|
send(res, 404, { 'content-type': 'text/plain; charset=utf-8' }, 'not_found');
|
||||||
|
}
|
||||||
|
|
||||||
|
function stripHopByHopHeaders(headers) {
|
||||||
|
const hop = new Set([
|
||||||
|
'connection',
|
||||||
|
'keep-alive',
|
||||||
|
'proxy-authenticate',
|
||||||
|
'proxy-authorization',
|
||||||
|
'te',
|
||||||
|
'trailer',
|
||||||
|
'transfer-encoding',
|
||||||
|
'upgrade',
|
||||||
|
]);
|
||||||
|
const out = {};
|
||||||
|
for (const [k, v] of Object.entries(headers || {})) {
|
||||||
|
if (hop.has(k.toLowerCase())) continue;
|
||||||
|
out[k] = v;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readHeader(req, name) {
|
||||||
|
const v = req.headers[String(name).toLowerCase()];
|
||||||
|
return Array.isArray(v) ? v[0] : v;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readCookie(req, name) {
|
||||||
|
const raw = typeof req.headers.cookie === 'string' ? req.headers.cookie : '';
|
||||||
|
if (!raw) return null;
|
||||||
|
const needle = `${name}=`;
|
||||||
|
for (const part of raw.split(';')) {
|
||||||
|
const t = part.trim();
|
||||||
|
if (!t.startsWith(needle)) continue;
|
||||||
|
return t.slice(needle.length) || null;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeDlobSource(value) {
|
||||||
|
const v = String(value ?? '')
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
if (v === 'mevnode') return 'mevnode';
|
||||||
|
if (v === 'drift') return 'drift';
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveDlobSource(req) {
|
||||||
|
const fromCookie = normalizeDlobSource(readCookie(req, DLOB_SOURCE_COOKIE));
|
||||||
|
if (fromCookie) return fromCookie;
|
||||||
|
return normalizeDlobSource(DLOB_SOURCE_DEFAULT) || 'mevnode';
|
||||||
|
}
|
||||||
|
|
||||||
|
function safeRedirectPath(value) {
|
||||||
|
const s = String(value ?? '').trim();
|
||||||
|
if (!s.startsWith('/')) return '/';
|
||||||
|
if (s.startsWith('//')) return '/';
|
||||||
|
return s.replace(/\r|\n/g, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
function setDlobSourceCookie(res, { secure, dlobSource }) {
|
||||||
|
const src = normalizeDlobSource(dlobSource);
|
||||||
|
if (!src) return false;
|
||||||
|
const parts = [
|
||||||
|
`${DLOB_SOURCE_COOKIE}=${src}`,
|
||||||
|
'Path=/',
|
||||||
|
'SameSite=Lax',
|
||||||
|
'HttpOnly',
|
||||||
|
'Max-Age=31536000',
|
||||||
|
];
|
||||||
|
if (secure) parts.push('Secure');
|
||||||
|
res.setHeader('set-cookie', parts.join('; '));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveAuthUser(req) {
|
||||||
|
const user = readHeader(req, AUTH_USER_HEADER) || readHeader(req, 'x-webauth-user');
|
||||||
|
const value = typeof user === 'string' ? user.trim() : '';
|
||||||
|
return value || null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isHttpsRequest(req) {
|
||||||
|
const xf = readHeader(req, 'x-forwarded-proto');
|
||||||
|
if (typeof xf === 'string' && xf.toLowerCase() === 'https') return true;
|
||||||
|
return Boolean(req.socket && req.socket.encrypted);
|
||||||
|
}
|
||||||
|
|
||||||
|
function base64urlEncode(buf) {
|
||||||
|
return Buffer.from(buf)
|
||||||
|
.toString('base64')
|
||||||
|
.replace(/\+/g, '-')
|
||||||
|
.replace(/\//g, '_')
|
||||||
|
.replace(/=+$/g, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
function base64urlDecode(str) {
|
||||||
|
const cleaned = String(str).replace(/-/g, '+').replace(/_/g, '/');
|
||||||
|
const pad = cleaned.length % 4 === 0 ? '' : '='.repeat(4 - (cleaned.length % 4));
|
||||||
|
return Buffer.from(cleaned + pad, 'base64');
|
||||||
|
}
|
||||||
|
|
||||||
|
function loadSessionSecret() {
|
||||||
|
if (process.env.AUTH_SESSION_SECRET && String(process.env.AUTH_SESSION_SECRET).trim()) {
|
||||||
|
return Buffer.from(String(process.env.AUTH_SESSION_SECRET).trim(), 'utf8');
|
||||||
|
}
|
||||||
|
if (AUTH_SESSION_SECRET_FILE) {
|
||||||
|
try {
|
||||||
|
const txt = readText(AUTH_SESSION_SECRET_FILE).trim();
|
||||||
|
if (txt) return Buffer.from(txt, 'utf8');
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return crypto.randomBytes(32);
|
||||||
|
}
|
||||||
|
|
||||||
|
const SESSION_SECRET = loadSessionSecret();
|
||||||
|
|
||||||
|
function signSessionPayload(payloadB64) {
|
||||||
|
return crypto.createHmac('sha256', SESSION_SECRET).update(payloadB64).digest();
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeSessionCookieValue(username) {
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
const exp = now + (Number.isFinite(AUTH_SESSION_TTL_SECONDS) && AUTH_SESSION_TTL_SECONDS > 0 ? AUTH_SESSION_TTL_SECONDS : 43200);
|
||||||
|
const payload = JSON.stringify({ u: String(username), exp });
|
||||||
|
const payloadB64 = base64urlEncode(Buffer.from(payload, 'utf8'));
|
||||||
|
const sigB64 = base64urlEncode(signSessionPayload(payloadB64));
|
||||||
|
return `${payloadB64}.${sigB64}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getSessionUser(req) {
|
||||||
|
const raw = readCookie(req, AUTH_SESSION_COOKIE);
|
||||||
|
if (!raw) return null;
|
||||||
|
const parts = raw.split('.');
|
||||||
|
if (parts.length !== 2) return null;
|
||||||
|
const [payloadB64, sigB64] = parts;
|
||||||
|
if (!payloadB64 || !sigB64) return null;
|
||||||
|
|
||||||
|
let payload;
|
||||||
|
try {
|
||||||
|
payload = JSON.parse(base64urlDecode(payloadB64).toString('utf8'));
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
const u = typeof payload?.u === 'string' ? payload.u.trim() : '';
|
||||||
|
const exp = Number(payload?.exp);
|
||||||
|
if (!u || !Number.isFinite(exp)) return null;
|
||||||
|
const now = Math.floor(Date.now() / 1000);
|
||||||
|
if (now >= exp) return null;
|
||||||
|
|
||||||
|
const expected = signSessionPayload(payloadB64);
|
||||||
|
let got;
|
||||||
|
try {
|
||||||
|
got = base64urlDecode(sigB64);
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (!timingSafeEqualBuf(expected, got)) return null;
|
||||||
|
|
||||||
|
return u;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveAuthenticatedUser(req) {
|
||||||
|
const sessionUser = getSessionUser(req);
|
||||||
|
if (sessionUser) return sessionUser;
|
||||||
|
const headerUser = resolveAuthUser(req);
|
||||||
|
if (headerUser) return headerUser;
|
||||||
|
if (AUTH_MODE === 'off' || AUTH_MODE === 'none' || AUTH_MODE === 'disabled') return 'anonymous';
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function clearSessionCookie(res, secure) {
|
||||||
|
const parts = [`${AUTH_SESSION_COOKIE}=`, 'Path=/', 'Max-Age=0', 'HttpOnly', 'SameSite=Lax'];
|
||||||
|
if (secure) parts.push('Secure');
|
||||||
|
res.setHeader('set-cookie', parts.join('; '));
|
||||||
|
}
|
||||||
|
|
||||||
|
function setSessionCookie(res, secure, username) {
|
||||||
|
const value = makeSessionCookieValue(username);
|
||||||
|
const parts = [
|
||||||
|
`${AUTH_SESSION_COOKIE}=${value}`,
|
||||||
|
'Path=/',
|
||||||
|
`Max-Age=${Number.isFinite(AUTH_SESSION_TTL_SECONDS) ? AUTH_SESSION_TTL_SECONDS : 43200}`,
|
||||||
|
'HttpOnly',
|
||||||
|
'SameSite=Lax',
|
||||||
|
];
|
||||||
|
if (secure) parts.push('Secure');
|
||||||
|
res.setHeader('set-cookie', parts.join('; '));
|
||||||
|
}
|
||||||
|
|
||||||
|
function verifyWithHtpasswd(username, password) {
|
||||||
|
try {
|
||||||
|
const r = spawnSync('htpasswd', ['-vb', HTPASSWD_FILE, String(username), String(password)], {
|
||||||
|
stdio: 'ignore',
|
||||||
|
timeout: 3000,
|
||||||
|
});
|
||||||
|
return r.status === 0;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function readBody(req, limitBytes = 1024 * 16) {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
let total = 0;
|
||||||
|
const chunks = [];
|
||||||
|
req.on('data', (chunk) => {
|
||||||
|
total += chunk.length;
|
||||||
|
if (total > limitBytes) {
|
||||||
|
reject(new Error('payload_too_large'));
|
||||||
|
req.destroy();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
chunks.push(chunk);
|
||||||
|
});
|
||||||
|
req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
||||||
|
req.on('error', reject);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function proxyApi(req, res, apiReadToken) {
|
||||||
|
const upstreamBase = new URL(API_UPSTREAM);
|
||||||
|
const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||||
|
|
||||||
|
const prefix = '/api';
|
||||||
|
const strippedPath = inUrl.pathname === prefix ? '/' : inUrl.pathname.startsWith(prefix + '/') ? inUrl.pathname.slice(prefix.length) : null;
|
||||||
|
if (strippedPath == null) {
|
||||||
|
send(res, 404, { 'content-type': 'text/plain; charset=utf-8' }, 'not_found');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const target = new URL(upstreamBase.toString());
|
||||||
|
target.pathname = strippedPath || '/';
|
||||||
|
target.search = inUrl.search;
|
||||||
|
|
||||||
|
const isHttps = target.protocol === 'https:';
|
||||||
|
const lib = isHttps ? https : http;
|
||||||
|
|
||||||
|
const headers = stripHopByHopHeaders(req.headers);
|
||||||
|
delete headers.authorization; // basic auth from client must not leak upstream
|
||||||
|
headers.host = target.host;
|
||||||
|
headers.authorization = `Bearer ${apiReadToken}`;
|
||||||
|
|
||||||
|
const upstreamReq = lib.request(
|
||||||
|
{
|
||||||
|
protocol: target.protocol,
|
||||||
|
hostname: target.hostname,
|
||||||
|
port: target.port || (isHttps ? 443 : 80),
|
||||||
|
method: req.method,
|
||||||
|
path: target.pathname + target.search,
|
||||||
|
headers,
|
||||||
|
},
|
||||||
|
(upstreamRes) => {
|
||||||
|
const outHeaders = stripHopByHopHeaders(upstreamRes.headers);
|
||||||
|
res.writeHead(upstreamRes.statusCode || 502, outHeaders);
|
||||||
|
upstreamRes.pipe(res);
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
upstreamReq.on('error', (err) => {
|
||||||
|
if (!res.headersSent) {
|
||||||
|
send(res, 502, { 'content-type': 'text/plain; charset=utf-8' }, `bad_gateway: ${err?.message || err}`);
|
||||||
|
} else {
|
||||||
|
res.destroy();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
req.pipe(upstreamReq);
|
||||||
|
}
|
||||||
|
|
||||||
|
function withCors(res) {
|
||||||
|
res.setHeader('access-control-allow-origin', GRAPHQL_CORS_ORIGIN);
|
||||||
|
res.setHeader('access-control-allow-methods', 'GET,POST,OPTIONS');
|
||||||
|
res.setHeader(
|
||||||
|
'access-control-allow-headers',
|
||||||
|
'content-type, authorization, x-hasura-admin-secret, x-hasura-role, x-hasura-user-id, x-hasura-dlob-source'
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function proxyGraphqlHttp(req, res) {
|
||||||
|
const upstreamBase = new URL(HASURA_UPSTREAM);
|
||||||
|
const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||||
|
|
||||||
|
const target = new URL(upstreamBase.toString());
|
||||||
|
target.pathname = HASURA_GRAPHQL_PATH;
|
||||||
|
target.search = inUrl.search;
|
||||||
|
|
||||||
|
const isHttps = target.protocol === 'https:';
|
||||||
|
const lib = isHttps ? https : http;
|
||||||
|
|
||||||
|
const headers = stripHopByHopHeaders(req.headers);
|
||||||
|
headers.host = target.host;
|
||||||
|
delete headers['x-hasura-dlob-source'];
|
||||||
|
headers['x-hasura-dlob-source'] = resolveDlobSource(req);
|
||||||
|
|
||||||
|
const upstreamReq = lib.request(
|
||||||
|
{
|
||||||
|
protocol: target.protocol,
|
||||||
|
hostname: target.hostname,
|
||||||
|
port: target.port || (isHttps ? 443 : 80),
|
||||||
|
method: req.method,
|
||||||
|
path: target.pathname + target.search,
|
||||||
|
headers,
|
||||||
|
},
|
||||||
|
(upstreamRes) => {
|
||||||
|
const outHeaders = stripHopByHopHeaders(upstreamRes.headers);
|
||||||
|
withCors(res);
|
||||||
|
res.writeHead(upstreamRes.statusCode || 502, outHeaders);
|
||||||
|
upstreamRes.pipe(res);
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
upstreamReq.on('error', (err) => {
|
||||||
|
if (!res.headersSent) {
|
||||||
|
withCors(res);
|
||||||
|
send(res, 502, { 'content-type': 'text/plain; charset=utf-8' }, `bad_gateway: ${err?.message || err}`);
|
||||||
|
} else {
|
||||||
|
res.destroy();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
req.pipe(upstreamReq);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isGraphqlPath(pathname) {
|
||||||
|
return pathname === '/graphql' || pathname === '/graphql-ws';
|
||||||
|
}
|
||||||
|
|
||||||
|
function proxyGraphqlWs(req, socket, head) {
|
||||||
|
const upstreamBase = new URL(HASURA_UPSTREAM);
|
||||||
|
const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||||
|
|
||||||
|
const target = new URL(upstreamBase.toString());
|
||||||
|
target.pathname = HASURA_GRAPHQL_PATH;
|
||||||
|
target.search = inUrl.search;
|
||||||
|
|
||||||
|
const port = Number(target.port || (target.protocol === 'https:' ? 443 : 80));
|
||||||
|
const host = target.hostname;
|
||||||
|
|
||||||
|
const connect =
|
||||||
|
target.protocol === 'https:'
|
||||||
|
? () => tls.connect({ host, port, servername: host })
|
||||||
|
: () => net.connect({ host, port });
|
||||||
|
|
||||||
|
const upstream = connect();
|
||||||
|
upstream.setNoDelay(true);
|
||||||
|
socket.setNoDelay(true);
|
||||||
|
|
||||||
|
// For WebSocket upgrades we must forward `connection`/`upgrade` and related headers.
|
||||||
|
const headers = { ...req.headers };
|
||||||
|
delete headers['content-length'];
|
||||||
|
delete headers['content-type'];
|
||||||
|
headers.host = target.host;
|
||||||
|
delete headers['x-hasura-dlob-source'];
|
||||||
|
headers['x-hasura-dlob-source'] = resolveDlobSource(req);
|
||||||
|
|
||||||
|
const lines = [];
|
||||||
|
lines.push(`GET ${target.pathname + target.search} HTTP/1.1`);
|
||||||
|
for (const [k, v] of Object.entries(headers)) {
|
||||||
|
if (v == null) continue;
|
||||||
|
if (Array.isArray(v)) {
|
||||||
|
for (const vv of v) lines.push(`${k}: ${vv}`);
|
||||||
|
} else {
|
||||||
|
lines.push(`${k}: ${v}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lines.push('', '');
|
||||||
|
upstream.write(lines.join('\r\n'));
|
||||||
|
|
||||||
|
if (head?.length) upstream.write(head);
|
||||||
|
|
||||||
|
upstream.on('error', () => {
|
||||||
|
try {
|
||||||
|
socket.destroy();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
});
|
||||||
|
socket.on('error', () => {
|
||||||
|
try {
|
||||||
|
upstream.destroy();
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
upstream.pipe(socket);
|
||||||
|
socket.pipe(upstream);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function handler(req, res) {
|
||||||
|
if (req.method === 'GET' && (req.url === '/healthz' || req.url?.startsWith('/healthz?'))) {
|
||||||
|
send(
|
||||||
|
res,
|
||||||
|
200,
|
||||||
|
{ 'content-type': 'application/json; charset=utf-8' },
|
||||||
|
JSON.stringify({ ok: true, version: APP_VERSION, buildTimestamp: BUILD_TIMESTAMP, startedAt: STARTED_AT })
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||||
|
if (isGraphqlPath(url.pathname)) {
|
||||||
|
if (req.method === 'OPTIONS') {
|
||||||
|
withCors(res);
|
||||||
|
res.statusCode = 204;
|
||||||
|
res.end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') {
|
||||||
|
const user = resolveAuthenticatedUser(req);
|
||||||
|
if (!user) {
|
||||||
|
withCors(res);
|
||||||
|
unauthorized(res);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
withCors(res);
|
||||||
|
proxyGraphqlHttp(req, res);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (req.method === 'GET' && url.pathname === '/whoami') {
|
||||||
|
sendJson(res, 200, { ok: true, user: resolveAuthenticatedUser(req), mode: AUTH_MODE });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.method === 'GET' && url.pathname === '/prefs/dlob-source') {
|
||||||
|
const set = url.searchParams.get('set');
|
||||||
|
if (!set) {
|
||||||
|
sendJson(res, 200, { ok: true, dlobSource: resolveDlobSource(req) });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ok = setDlobSourceCookie(res, { secure: isHttpsRequest(req), dlobSource: set });
|
||||||
|
if (!ok) {
|
||||||
|
sendJson(res, 400, { ok: false, error: 'invalid_dlob_source' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.statusCode = 302;
|
||||||
|
res.setHeader('location', safeRedirectPath(url.searchParams.get('redirect') || '/'));
|
||||||
|
res.end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.method === 'POST' && url.pathname === '/auth/login') {
|
||||||
|
if (AUTH_MODE === 'off' || AUTH_MODE === 'none' || AUTH_MODE === 'disabled') {
|
||||||
|
sendJson(res, 400, { ok: false, error: 'auth_disabled' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const raw = await readBody(req);
|
||||||
|
const ct = String(req.headers['content-type'] || '').toLowerCase();
|
||||||
|
let username = '';
|
||||||
|
let password = '';
|
||||||
|
if (ct.includes('application/json')) {
|
||||||
|
let json;
|
||||||
|
try {
|
||||||
|
json = JSON.parse(raw);
|
||||||
|
} catch {
|
||||||
|
sendJson(res, 400, { ok: false, error: 'bad_json' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
username = typeof json?.username === 'string' ? json.username.trim() : '';
|
||||||
|
password = typeof json?.password === 'string' ? json.password : '';
|
||||||
|
} else {
|
||||||
|
const params = new URLSearchParams(raw);
|
||||||
|
username = String(params.get('username') || '').trim();
|
||||||
|
password = String(params.get('password') || '');
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!username || !password || username.length > 64 || password.length > 200) {
|
||||||
|
sendJson(res, 400, { ok: false, error: 'invalid_input' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ok = verifyWithHtpasswd(username, password);
|
||||||
|
if (!ok) {
|
||||||
|
unauthorized(res);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const secure = isHttpsRequest(req);
|
||||||
|
setSessionCookie(res, secure, username);
|
||||||
|
sendJson(res, 200, { ok: true, user: username });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((req.method === 'POST' || req.method === 'GET') && (url.pathname === '/auth/logout' || url.pathname === '/logout')) {
|
||||||
|
clearSessionCookie(res, isHttpsRequest(req));
|
||||||
|
if (req.method === 'GET') {
|
||||||
|
res.statusCode = 302;
|
||||||
|
res.setHeader('location', '/');
|
||||||
|
res.end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
sendJson(res, 200, { ok: true });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (BASIC_AUTH_ENABLED) {
|
||||||
|
let creds;
|
||||||
|
try {
|
||||||
|
creds = loadBasicAuth();
|
||||||
|
} catch (e) {
|
||||||
|
send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!isAuthorized(req, creds)) {
|
||||||
|
basicAuthRequired(res);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (req.url?.startsWith('/api') && (req.url === '/api' || req.url.startsWith('/api/'))) {
|
||||||
|
if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') {
|
||||||
|
const user = resolveAuthenticatedUser(req);
|
||||||
|
if (!user) {
|
||||||
|
unauthorized(res);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let token;
|
||||||
|
try {
|
||||||
|
token = loadApiReadToken();
|
||||||
|
} catch (e) {
|
||||||
|
send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
proxyApi(req, res, token);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
serveStatic(req, res);
|
||||||
|
}
|
||||||
|
|
||||||
|
const server = http.createServer((req, res) => {
|
||||||
|
handler(req, res).catch((e) => {
|
||||||
|
if (res.headersSent) {
|
||||||
|
res.destroy();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e));
|
||||||
|
});
|
||||||
|
});
|
||||||
|
server.on('upgrade', (req, socket, head) => {
|
||||||
|
try {
|
||||||
|
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||||
|
if (!isGraphqlPath(url.pathname)) {
|
||||||
|
socket.destroy();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') {
|
||||||
|
const user = resolveAuthenticatedUser(req);
|
||||||
|
if (!user) {
|
||||||
|
try {
|
||||||
|
socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
|
||||||
|
} catch {
|
||||||
|
// ignore
|
||||||
|
}
|
||||||
|
socket.destroy();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
proxyGraphqlWs(req, socket, head);
|
||||||
|
} catch {
|
||||||
|
socket.destroy();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
server.listen(PORT, () => {
|
||||||
|
console.log(
|
||||||
|
JSON.stringify(
|
||||||
|
{
|
||||||
|
service: 'trade-frontend',
|
||||||
|
port: PORT,
|
||||||
|
staticDir: STATIC_DIR,
|
||||||
|
apiUpstream: API_UPSTREAM,
|
||||||
|
hasuraUpstream: HASURA_UPSTREAM,
|
||||||
|
basicAuthFile: BASIC_AUTH_FILE,
|
||||||
|
basicAuthMode: BASIC_AUTH_MODE,
|
||||||
|
apiReadTokenFile: API_READ_TOKEN_FILE,
|
||||||
|
authUserHeader: AUTH_USER_HEADER,
|
||||||
|
authMode: AUTH_MODE,
|
||||||
|
htpasswdFile: HTPASSWD_FILE,
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2
|
||||||
|
)
|
||||||
|
);
|
||||||
|
});
|
||||||
@@ -6,5 +6,18 @@ namespace: trade-prod
|
|||||||
resources:
|
resources:
|
||||||
- ../../base
|
- ../../base
|
||||||
|
|
||||||
|
patchesStrategicMerge:
|
||||||
|
- dlob-rpc-endpoint-patch.yaml
|
||||||
|
- dlob-rpc-server-endpoint-patch.yaml
|
||||||
|
- frontend-graphql-proxy-patch.yaml
|
||||||
|
|
||||||
|
configMapGenerator:
|
||||||
|
- name: trade-frontend-server-script
|
||||||
|
files:
|
||||||
|
- frontend-server.mjs
|
||||||
|
|
||||||
|
generatorOptions:
|
||||||
|
disableNameSuffixHash: true
|
||||||
|
|
||||||
commonLabels:
|
commonLabels:
|
||||||
env: prod
|
env: prod
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ function resolveConfig() {
|
|||||||
const hasuraAdminSecret = envString('HASURA_ADMIN_SECRET', '');
|
const hasuraAdminSecret = envString('HASURA_ADMIN_SECRET', '');
|
||||||
if (!hasuraAdminSecret) throw new Error('Missing HASURA_ADMIN_SECRET');
|
if (!hasuraAdminSecret) throw new Error('Missing HASURA_ADMIN_SECRET');
|
||||||
|
|
||||||
const markets = envList('DLOB_MARKETS', 'PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP');
|
const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
|
||||||
const pollMs = envInt('TICKS_POLL_MS', 1000, { min: 250, max: 60_000 });
|
const pollMs = envInt('TICKS_POLL_MS', 1000, { min: 250, max: 60_000 });
|
||||||
const source = envString('TICKS_SOURCE', 'dlob_stats');
|
const source = envString('TICKS_SOURCE', 'dlob_stats');
|
||||||
|
|
||||||
|
|||||||
14
kustomize/overlays/staging/dlob-rpc-endpoint-patch.yaml
Normal file
14
kustomize/overlays/staging/dlob-rpc-endpoint-patch.yaml
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: dlob-publisher
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: publisher
|
||||||
|
env:
|
||||||
|
- name: ENDPOINT
|
||||||
|
value: "http://10.66.66.1:8899"
|
||||||
|
- name: WS_ENDPOINT
|
||||||
|
value: "ws://10.66.66.1:8900"
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: dlob-server
|
||||||
|
spec:
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: server
|
||||||
|
env:
|
||||||
|
- name: ENDPOINT
|
||||||
|
value: "http://10.66.66.1:8899"
|
||||||
|
- name: WS_ENDPOINT
|
||||||
|
value: "ws://10.66.66.1:8900"
|
||||||
@@ -1,16 +0,0 @@
|
|||||||
apiVersion: apps/v1
|
|
||||||
kind: Deployment
|
|
||||||
metadata:
|
|
||||||
name: dlob-worker
|
|
||||||
spec:
|
|
||||||
template:
|
|
||||||
spec:
|
|
||||||
hostNetwork: true
|
|
||||||
dnsPolicy: ClusterFirstWithHostNet
|
|
||||||
containers:
|
|
||||||
- name: worker
|
|
||||||
env:
|
|
||||||
- name: DLOB_HTTP_URL
|
|
||||||
value: https://dlob.drift.trade
|
|
||||||
- name: DLOB_FORCE_IPV6
|
|
||||||
value: "true"
|
|
||||||
@@ -37,6 +37,8 @@ const AUTH_SESSION_COOKIE = String(process.env.AUTH_SESSION_COOKIE || 'trade_ses
|
|||||||
.trim()
|
.trim()
|
||||||
.toLowerCase();
|
.toLowerCase();
|
||||||
const AUTH_SESSION_TTL_SECONDS = Number.parseInt(process.env.AUTH_SESSION_TTL_SECONDS || '43200', 10); // 12h
|
const AUTH_SESSION_TTL_SECONDS = Number.parseInt(process.env.AUTH_SESSION_TTL_SECONDS || '43200', 10); // 12h
|
||||||
|
const DLOB_SOURCE_COOKIE = String(process.env.DLOB_SOURCE_COOKIE || 'trade_dlob_source').trim() || 'trade_dlob_source';
|
||||||
|
const DLOB_SOURCE_DEFAULT = String(process.env.DLOB_SOURCE_DEFAULT || 'mevnode').trim() || 'mevnode';
|
||||||
|
|
||||||
function readJson(filePath) {
|
function readJson(filePath) {
|
||||||
const raw = fs.readFileSync(filePath, 'utf8');
|
const raw = fs.readFileSync(filePath, 'utf8');
|
||||||
@@ -143,6 +145,29 @@ function safePathFromUrlPath(urlPath) {
|
|||||||
return normalized;
|
return normalized;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function injectIndexHtml(html, { dlobSource, redirectPath }) {
|
||||||
|
const src = normalizeDlobSource(dlobSource) || 'mevnode';
|
||||||
|
const redirect = safeRedirectPath(redirectPath);
|
||||||
|
const hrefBase = `/prefs/dlob-source?redirect=${encodeURIComponent(redirect)}&set=`;
|
||||||
|
|
||||||
|
const styleActive = 'font-weight:700;text-decoration:underline;';
|
||||||
|
const styleInactive = 'font-weight:400;text-decoration:none;';
|
||||||
|
|
||||||
|
const snippet = `
|
||||||
|
<!-- trade: dlob source switch -->
|
||||||
|
<div style="position:fixed;right:12px;bottom:12px;z-index:2147483647;background:rgba(0,0,0,0.72);color:#fff;padding:8px 10px;border-radius:10px;font:12px/1.2 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;backdrop-filter:blur(6px);">
|
||||||
|
<span style="opacity:0.85;margin-right:6px;">DLOB</span>
|
||||||
|
<a href="${hrefBase}mevnode" style="color:#fff;${src === 'mevnode' ? styleActive : styleInactive}">mevnode</a>
|
||||||
|
<span style="opacity:0.6;margin:0 6px;">|</span>
|
||||||
|
<a href="${hrefBase}drift" style="color:#fff;${src === 'drift' ? styleActive : styleInactive}">drift</a>
|
||||||
|
</div>
|
||||||
|
`;
|
||||||
|
|
||||||
|
const bodyClose = /<\/body>/i;
|
||||||
|
if (bodyClose.test(html)) return html.replace(bodyClose, `${snippet}</body>`);
|
||||||
|
return `${html}\n${snippet}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
function serveStatic(req, res) {
|
function serveStatic(req, res) {
|
||||||
if (req.method !== 'GET' && req.method !== 'HEAD') {
|
if (req.method !== 'GET' && req.method !== 'HEAD') {
|
||||||
send(res, 405, { 'content-type': 'text/plain; charset=utf-8' }, 'method_not_allowed');
|
send(res, 405, { 'content-type': 'text/plain; charset=utf-8' }, 'method_not_allowed');
|
||||||
@@ -171,6 +196,15 @@ function serveStatic(req, res) {
|
|||||||
res.setHeader('content-type', contentTypeFor(filePath));
|
res.setHeader('content-type', contentTypeFor(filePath));
|
||||||
res.setHeader('cache-control', filePath.endsWith('index.html') ? 'no-cache' : 'public, max-age=31536000');
|
res.setHeader('cache-control', filePath.endsWith('index.html') ? 'no-cache' : 'public, max-age=31536000');
|
||||||
if (req.method === 'HEAD') return void res.end();
|
if (req.method === 'HEAD') return void res.end();
|
||||||
|
if (filePath.endsWith('index.html')) {
|
||||||
|
const html = fs.readFileSync(filePath, 'utf8');
|
||||||
|
const injected = injectIndexHtml(html, {
|
||||||
|
dlobSource: resolveDlobSource(req),
|
||||||
|
redirectPath: url.pathname + url.search,
|
||||||
|
});
|
||||||
|
res.end(injected);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
fs.createReadStream(filePath).pipe(res);
|
fs.createReadStream(filePath).pipe(res);
|
||||||
return true;
|
return true;
|
||||||
} catch {
|
} catch {
|
||||||
@@ -222,6 +256,43 @@ function readCookie(req, name) {
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeDlobSource(value) {
|
||||||
|
const v = String(value ?? '')
|
||||||
|
.trim()
|
||||||
|
.toLowerCase();
|
||||||
|
if (v === 'mevnode') return 'mevnode';
|
||||||
|
if (v === 'drift') return 'drift';
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveDlobSource(req) {
|
||||||
|
const fromCookie = normalizeDlobSource(readCookie(req, DLOB_SOURCE_COOKIE));
|
||||||
|
if (fromCookie) return fromCookie;
|
||||||
|
return normalizeDlobSource(DLOB_SOURCE_DEFAULT) || 'mevnode';
|
||||||
|
}
|
||||||
|
|
||||||
|
function safeRedirectPath(value) {
|
||||||
|
const s = String(value ?? '').trim();
|
||||||
|
if (!s.startsWith('/')) return '/';
|
||||||
|
if (s.startsWith('//')) return '/';
|
||||||
|
return s.replace(/\r|\n/g, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
function setDlobSourceCookie(res, { secure, dlobSource }) {
|
||||||
|
const src = normalizeDlobSource(dlobSource);
|
||||||
|
if (!src) return false;
|
||||||
|
const parts = [
|
||||||
|
`${DLOB_SOURCE_COOKIE}=${src}`,
|
||||||
|
'Path=/',
|
||||||
|
'SameSite=Lax',
|
||||||
|
'HttpOnly',
|
||||||
|
'Max-Age=31536000',
|
||||||
|
];
|
||||||
|
if (secure) parts.push('Secure');
|
||||||
|
res.setHeader('set-cookie', parts.join('; '));
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
function resolveAuthUser(req) {
|
function resolveAuthUser(req) {
|
||||||
const user = readHeader(req, AUTH_USER_HEADER) || readHeader(req, 'x-webauth-user');
|
const user = readHeader(req, AUTH_USER_HEADER) || readHeader(req, 'x-webauth-user');
|
||||||
const value = typeof user === 'string' ? user.trim() : '';
|
const value = typeof user === 'string' ? user.trim() : '';
|
||||||
@@ -423,7 +494,7 @@ function withCors(res) {
|
|||||||
res.setHeader('access-control-allow-methods', 'GET,POST,OPTIONS');
|
res.setHeader('access-control-allow-methods', 'GET,POST,OPTIONS');
|
||||||
res.setHeader(
|
res.setHeader(
|
||||||
'access-control-allow-headers',
|
'access-control-allow-headers',
|
||||||
'content-type, authorization, x-hasura-admin-secret, x-hasura-role, x-hasura-user-id'
|
'content-type, authorization, x-hasura-admin-secret, x-hasura-role, x-hasura-user-id, x-hasura-dlob-source'
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -440,6 +511,8 @@ function proxyGraphqlHttp(req, res) {
|
|||||||
|
|
||||||
const headers = stripHopByHopHeaders(req.headers);
|
const headers = stripHopByHopHeaders(req.headers);
|
||||||
headers.host = target.host;
|
headers.host = target.host;
|
||||||
|
delete headers['x-hasura-dlob-source'];
|
||||||
|
headers['x-hasura-dlob-source'] = resolveDlobSource(req);
|
||||||
|
|
||||||
const upstreamReq = lib.request(
|
const upstreamReq = lib.request(
|
||||||
{
|
{
|
||||||
@@ -499,6 +572,8 @@ function proxyGraphqlWs(req, socket, head) {
|
|||||||
delete headers['content-length'];
|
delete headers['content-length'];
|
||||||
delete headers['content-type'];
|
delete headers['content-type'];
|
||||||
headers.host = target.host;
|
headers.host = target.host;
|
||||||
|
delete headers['x-hasura-dlob-source'];
|
||||||
|
headers['x-hasura-dlob-source'] = resolveDlobSource(req);
|
||||||
|
|
||||||
const lines = [];
|
const lines = [];
|
||||||
lines.push(`GET ${target.pathname + target.search} HTTP/1.1`);
|
lines.push(`GET ${target.pathname + target.search} HTTP/1.1`);
|
||||||
@@ -570,6 +645,25 @@ async function handler(req, res) {
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (req.method === 'GET' && url.pathname === '/prefs/dlob-source') {
|
||||||
|
const set = url.searchParams.get('set');
|
||||||
|
if (!set) {
|
||||||
|
sendJson(res, 200, { ok: true, dlobSource: resolveDlobSource(req) });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const ok = setDlobSourceCookie(res, { secure: isHttpsRequest(req), dlobSource: set });
|
||||||
|
if (!ok) {
|
||||||
|
sendJson(res, 400, { ok: false, error: 'invalid_dlob_source' });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
res.statusCode = 302;
|
||||||
|
res.setHeader('location', safeRedirectPath(url.searchParams.get('redirect') || '/'));
|
||||||
|
res.end();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (req.method === 'POST' && url.pathname === '/auth/login') {
|
if (req.method === 'POST' && url.pathname === '/auth/login') {
|
||||||
if (AUTH_MODE === 'off' || AUTH_MODE === 'none' || AUTH_MODE === 'disabled') {
|
if (AUTH_MODE === 'off' || AUTH_MODE === 'none' || AUTH_MODE === 'disabled') {
|
||||||
sendJson(res, 400, { ok: false, error: 'auth_disabled' });
|
sendJson(res, 400, { ok: false, error: 'auth_disabled' });
|
||||||
|
|||||||
@@ -18,7 +18,7 @@ spec:
|
|||||||
name: trade-hasura
|
name: trade-hasura
|
||||||
key: HASURA_GRAPHQL_ADMIN_SECRET
|
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||||
- name: DLOB_MARKETS
|
- name: DLOB_MARKETS
|
||||||
value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
|
value: SOL-PERP,DOGE-PERP,JUP-PERP
|
||||||
- name: TICKS_POLL_MS
|
- name: TICKS_POLL_MS
|
||||||
value: "1000"
|
value: "1000"
|
||||||
- name: TICKS_SOURCE
|
- name: TICKS_SOURCE
|
||||||
|
|||||||
@@ -10,11 +10,12 @@ resources:
|
|||||||
- frontend-ingress-root.yaml
|
- frontend-ingress-root.yaml
|
||||||
|
|
||||||
patchesStrategicMerge:
|
patchesStrategicMerge:
|
||||||
|
- dlob-rpc-endpoint-patch.yaml
|
||||||
|
- dlob-rpc-server-endpoint-patch.yaml
|
||||||
- hasura-patch.yaml
|
- hasura-patch.yaml
|
||||||
- frontend-auth-patch.yaml
|
- frontend-auth-patch.yaml
|
||||||
- frontend-graphql-proxy-patch.yaml
|
- frontend-graphql-proxy-patch.yaml
|
||||||
- ingestor-dlob-patch.yaml
|
- ingestor-dlob-patch.yaml
|
||||||
- dlob-worker-patch.yaml
|
|
||||||
|
|
||||||
configMapGenerator:
|
configMapGenerator:
|
||||||
- name: trade-dlob-ingestor-script
|
- name: trade-dlob-ingestor-script
|
||||||
|
|||||||
Reference in New Issue
Block a user