92 Commits

Author SHA1 Message Date
mpabi
108bf22968 chore(kustomize): remove dlob rpc endpoint patches 2026-03-12 23:26:17 +01:00
mpabi
59c3f3ee06 feat(dlob): pin markets and wire mevnode endpoints
- Limit DLOB workers/ingestors to SOL-PERP, DOGE-PERP, JUP-PERP across base and staging config.
- Set publisher market ids to [0,7,24] for drift protocol.
- Add overlay patches for dlob-publisher and dlob-server to use wg0 RPC endpoints 10.66.66.1:8899/8900 in staging and prod.
- Extend Agave dashboard and add PrometheusRules for RPC up/lag/I/O alerts.
- Ensure overlays reference new patches for automated ArgoCD rollouts.
2026-02-15 00:40:50 +01:00
mpabi
9c4c3096d7 chore(grafana): set default time range to now 2026-02-14 14:08:11 +01:00
mpabi
b02bd6b66c feat(grafana): expand agave dashboard with node + geyser stats 2026-02-14 14:01:34 +01:00
mpabi
890ac4c86e feat(monitoring): add mpabi node-exporter scrape and agave dashboard 2026-02-14 13:43:26 +01:00
u1
9c6e974d3a fix(monitoring): add fallback_scrape_protocol for geyser metrics (missing Content-Type) 2026-02-14 11:52:50 +00:00
u1
d1dc32d3bb chore(monitoring): delete traefik basic auth secret 2026-02-14 11:28:40 +00:00
u1
a6a0accd6a chore(monitoring): remove basic auth middleware from prometheus ingressroute 2026-02-14 11:28:39 +00:00
u1
3b8b1f5492 chore(monitoring): delete traefik basic auth middleware 2026-02-14 11:28:38 +00:00
u1
0c80a08732 chore(monitoring): remove basic auth middleware from grafana ingressroute 2026-02-14 11:28:38 +00:00
u1
8b72e62621 chore(monitoring): remove traefik basic auth from monitoring extras 2026-02-14 11:28:37 +00:00
u1
ff7a4b69cd fix(monitoring): disable prometheus-operator admission webhooks for GitOps stability 2026-02-14 11:04:31 +00:00
u1
e7f9594381 fix(monitoring): use Replace sync option to avoid CRD patch annotation size issues 2026-02-14 11:01:03 +00:00
u1
0c0f219d02 fix(monitoring): skip CRDs in ArgoCD helm rendering 2026-02-14 11:00:04 +00:00
u1
9cf0ed84d9 fix(monitoring): disable CRD management in ArgoCD (avoid annotation size limit) 2026-02-14 10:59:32 +00:00
u1
2702edce22 fix(monitoring): keep monitoring-stack as helm release name (avoid duplicate install) 2026-02-14 10:58:11 +00:00
u1
c33533fcd6 fix(monitoring): route prometheus-http to monitoring-stack service 2026-02-14 10:58:10 +00:00
u1
32eb047551 fix(monitoring): route prometheus to monitoring-stack service 2026-02-14 10:58:09 +00:00
u1
d27d64e407 fix(monitoring): route grafana-http to monitoring-stack service 2026-02-14 10:58:08 +00:00
u1
fa6893aa98 fix(monitoring): route grafana to monitoring-stack service 2026-02-14 10:58:08 +00:00
u1
77a8265b40 fix(monitoring): set helm releaseName to reuse existing kube-prometheus-stack resources 2026-02-14 10:55:36 +00:00
u1
c692f8d653 fix(monitoring): use server-side apply to avoid CRD annotation size limit 2026-02-14 10:54:48 +00:00
u1
47096c9877 feat(monitoring): add ArgoCD app for monitoring ingress/certs/auth 2026-02-14 10:53:55 +00:00
u1
0104532e73 feat(monitoring): add ArgoCD app for kube-prometheus-stack 2026-02-14 10:53:55 +00:00
u1
19e7e48190 feat(monitoring): add prometheus ingressroute http->https redirect 2026-02-14 10:53:39 +00:00
u1
7ef3ffe62c feat(monitoring): add prometheus ingressroute (https + basic auth) 2026-02-14 10:53:38 +00:00
u1
1853ef6452 feat(monitoring): add grafana ingressroute http->https redirect 2026-02-14 10:53:38 +00:00
u1
34ef9490a4 feat(monitoring): add grafana ingressroute (https + basic auth) 2026-02-14 10:53:37 +00:00
u1
f3bc3da9bb feat(monitoring): add TLS certificate for grafana/prometheus 2026-02-14 10:53:35 +00:00
u1
f797234abd feat(monitoring): add traefik redirect-to-https middleware 2026-02-14 10:53:35 +00:00
u1
c95a4286fb feat(monitoring): add basic auth secret for traefik 2026-02-14 10:53:34 +00:00
u1
b72f281651 feat(monitoring): add traefik basic auth middleware 2026-02-14 10:53:34 +00:00
u1
98912c5b03 feat(monitoring): add kustomize scaffold for monitoring extras 2026-02-14 10:53:05 +00:00
u1
28876fa1d2 fix(dlob): relax dlob-publisher probes 2026-02-13 12:04:13 +01:00
u1
5f46d26037 feat(frontend): per-user DLOB source header
- Add cookie-based source selector (mevnode|drift)\n- Proxy sets x-hasura-dlob-source for HTTP + WS\n- Include same proxy script mount in prod overlay
2026-02-13 11:33:13 +01:00
u1
57433c7e75 feat(dlob): support two sources + per-user switch
- Add "source" column + composite PKs for DLOB tables\n- Filter public Hasura selects by X-Hasura-Dlob-Source\n- Run parallel workers for mevnode + dlob.drift.trade\n- Frontend proxy sets x-hasura-dlob-source from cookie and injects UI switch
2026-02-13 11:25:32 +01:00
u1
9e7d7b88ac feat(candles): support oracle OHLC basis 2026-02-03 12:51:29 +01:00
u1
bd88eaa3c8 fix(api): keep candle open continuity 2026-02-03 10:27:54 +01:00
u1
cd4cbec7e0 fix(api): fill-forward missing candle buckets 2026-02-02 23:14:18 +01:00
u1
144f6e7c86 fix(db): fill-forward candles buckets 2026-02-02 23:05:04 +01:00
u1
ef8f7cbeaa fix(candles): serve chart from cache and stabilize any-source 2026-02-02 22:28:30 +01:00
codex
507da3165f chore(frontend): bump image sha-b06fe7f 2026-02-01 22:00:55 +01:00
codex
f41792a0fa fix(candles-cache): cap per-tf window to 1024 2026-02-01 19:07:49 +01:00
codex
e16a02453e fix(staging): hoist ensurePublicSelectTable in hasura bootstrap 2026-02-01 18:49:00 +01:00
codex
b46d7d85c6 chore(staging): add candles-cache-worker deployment 2026-02-01 18:13:24 +01:00
codex
e5543f408a feat(staging): add candles-cache-worker and api override 2026-02-01 18:12:26 +01:00
codex
b239f564b2 feat(staging): add candles cache + v2 slippage 2026-02-01 18:12:15 +01:00
u1
e6a2731d7e feat(api): run wrapper to add contract cost endpoints 2026-02-01 00:42:58 +00:00
u1
bb2db662d5 feat(api): add trade-api wrapper configmap 2026-02-01 00:42:27 +00:00
u1
84aade8c89 feat(api): add wrapper to extend trade-api endpoints without rebuilding image 2026-02-01 00:41:59 +00:00
u1
2e77fe85e4 chore(dlob): add small slippage sizes for 10 USD estimates 2026-02-01 00:00:21 +00:00
u1
64ea940f30 feat(staging): add dlob ts history archiver 2026-01-31 13:02:10 +01:00
u1
a5ef8b5c46 staging: allow trade.mpabi.pl origin for Hasura WS 2026-01-15 20:29:06 +00:00
u1
2451e1b046 staging: wire /graphql proxy into trade-frontend 2026-01-14 08:49:50 +00:00
u1
63288a2255 staging: mount frontend server script for /graphql 2026-01-14 08:49:27 +00:00
u1
4657e766f1 staging: add frontend server script with /graphql proxy 2026-01-14 08:49:17 +00:00
u1
d3f00cfbb4 chore(staging): bump trade-frontend image to sha-ob-20260111203413 2026-01-11 20:34:54 +00:00
u1
773784e1a3 chore(staging): bump trade-api image to k3s-20260111095435 2026-01-11 09:55:33 +00:00
u1
56045f8f55 chore(staging): bump trade-frontend image to sha-ca9e44a 2026-01-11 09:55:18 +00:00
u1
1ba4c72e11 feat(staging): switch ticks ingest to dlob stats 2026-01-10 12:16:17 +00:00
u1
1efa41c112 feat(staging): switch ingestor to dlob stats 2026-01-10 12:16:09 +00:00
u1
b925ad78e8 feat(staging): ingest ticks from dlob stats 2026-01-10 12:16:01 +00:00
u1
0f638f1d70 feat(deploy): add dlob derived stats workers 2026-01-10 11:45:01 +00:00
u1
5c59b27808 feat(dlob): add dlob-slippage-worker script 2026-01-10 11:30:41 +00:00
u1
d66ce89c7a feat(dlob): add dlob-slippage-worker deployment 2026-01-10 11:28:12 +00:00
u1
78711b9baf feat(dlob): add dlob-depth-worker script 2026-01-10 11:27:57 +00:00
u1
51424eeefc feat(dlob): add dlob-depth-worker deployment 2026-01-10 11:26:41 +00:00
u1
45a637f545 feat(hasura): track derived dlob stats tables 2026-01-10 11:26:13 +00:00
u1
1b1603c8f0 feat(db): add dlob depth/slippage tables 2026-01-10 11:24:51 +00:00
u1
86fcc286e5 fix(dlob): run redis as single-node cluster 2026-01-10 10:08:48 +00:00
u1
5992a54ac3 fix(dlob-worker): allow forcing IPv6 egress 2026-01-10 10:08:32 +00:00
u1
44853ab6f6 chore(staging): enable dlob-worker ipv6 patch 2026-01-10 10:07:20 +00:00
u1
c06a459b67 fix(staging): force dlob-worker via IPv6 2026-01-10 10:07:12 +00:00
u1
370cb3f74c fix(staging): disable redis TLS for dlob 2026-01-10 09:49:38 +01:00
u1
ead68a25cf feat(staging): self-host DLOB service 2026-01-10 09:37:52 +01:00
u1
392458ad99 fix(hasura): ignore already-untracked errors in bootstrap 2026-01-10 01:16:24 +00:00
u1
bd05eab467 fix(deploy): bump frontend image to sha-226406a 2026-01-10 01:06:22 +00:00
u1
f39f201b70 feat(dlob): wire worker + migrate job into kustomize 2026-01-10 00:54:04 +00:00
u1
fee9120bc2 feat(dlob): add worker script 2026-01-10 00:53:48 +00:00
u1
f32be5ea1c feat(dlob): add dlob worker deployment 2026-01-10 00:52:45 +00:00
u1
1a7a1c4de8 chore(hasura): run bootstrap as Argo hook 2026-01-10 00:52:33 +00:00
u1
a628f9044f feat(db): add postgres migrate job 2026-01-10 00:52:10 +00:00
u1
0c853354eb feat(hasura): track DLOB tables and public permissions 2026-01-10 00:42:46 +00:00
u1
476eb331c2 feat(db): add DLOB latest tables 2026-01-10 00:41:53 +00:00
u1
93587645cd feat(hasura): enable unauthorized public role 2026-01-10 00:41:09 +00:00
u1
42f26089e1 deploy(frontend): bump image to sha-f85e6da 2026-01-10 00:40:46 +00:00
u1
75e87a7cc8 feat(staging): tune fake-ingestor realism
- Seed uses paged /v1/ticks fetch (beyond 5k) to capture longer history.

- Longer return blocks + adjusted volatility params for a more lifelike curve.
2026-01-09 01:48:55 +01:00
u1
0eef6bca12 feat(staging): switch ingestor to fake
- Replaces Drift/RPC ingest with a seeded fake generator in the staging overlay.

- Seeds from existing ticks via trade-api /v1/ticks and writes new ticks via /v1/ingest/tick.
2026-01-09 01:13:35 +01:00
u1
ac50ca2117 chore(staging): bump images + add mpabi.pl host
- Switch custom images to gitea.mpabi.pl registry (rv32i.pl TLS no longer matches).
- Bump frontend to sha-a12c86f.
- Add staging ingress for mpabi.pl pointing to trade-frontend.
2026-01-06 23:46:06 +01:00
u1
5442d52ab1 chore(argocd): use gitea.mpabi.pl repoURL 2026-01-06 18:10:58 +01:00
u1
227d035f01 chore(ingress): switch hosts to mpabi.pl
- trade: trade.mpabi.pl\n- portainer: portainer.mpabi.pl
2026-01-06 18:03:42 +01:00
u1
f1da4ca774 chore(frontend): bump image to sha-6107c4e 2026-01-06 17:50:31 +01:00
57 changed files with 9095 additions and 25 deletions

View File

@@ -0,0 +1,20 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: monitoring-extras
namespace: argocd
spec:
project: default
source:
repoURL: https://gitea.mpabi.pl/trade/trade-deploy.git
targetRevision: main
path: kustomize/infra/monitoring-extras
destination:
server: https://kubernetes.default.svc
namespace: monitoring
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true

View File

@@ -0,0 +1,50 @@
apiVersion: argoproj.io/v1alpha1
kind: Application
metadata:
name: monitoring-stack
namespace: argocd
spec:
project: default
source:
repoURL: https://prometheus-community.github.io/helm-charts
chart: kube-prometheus-stack
targetRevision: 81.6.9
helm:
skipCrds: true
values: |
grafana:
enabled: true
prometheus:
prometheusSpec:
scrapeInterval: 15s
evaluationInterval: 15s
additionalScrapeConfigs:
- job_name: mpabi-yellowstone-geyser
metrics_path: /metrics
scrape_interval: 10s
fallback_scrape_protocol: PrometheusText0.0.4
static_configs:
- targets:
- 10.66.66.1:8999
- job_name: mpabi-node-exporter
metrics_path: /metrics
scrape_interval: 15s
static_configs:
- targets:
- 10.66.66.1:9100
prometheusOperator:
admissionWebhooks:
enabled: false
patch:
enabled: false
destination:
server: https://kubernetes.default.svc
namespace: monitoring
syncPolicy:
automated:
prune: true
selfHeal: true
syncOptions:
- CreateNamespace=true
- Replace=true
- ServerSideApply=true

View File

@@ -6,7 +6,7 @@ metadata:
spec:
project: default
source:
repoURL: https://rv32i.pl/trade/trade-deploy.git
repoURL: https://gitea.mpabi.pl/trade/trade-deploy.git
targetRevision: main
path: kustomize/infra/portainer
destination:

View File

@@ -6,7 +6,7 @@ metadata:
spec:
project: default
source:
repoURL: https://rv32i.pl/trade/trade-deploy.git
repoURL: https://gitea.mpabi.pl/trade/trade-deploy.git
targetRevision: main
path: kustomize/overlays/prod
destination:

View File

@@ -6,7 +6,7 @@ metadata:
spec:
project: default
source:
repoURL: https://rv32i.pl/trade/trade-deploy.git
repoURL: https://gitea.mpabi.pl/trade/trade-deploy.git
targetRevision: main
path: kustomize/overlays/staging
destination:

View File

@@ -14,16 +14,39 @@ spec:
spec:
imagePullSecrets:
- name: gitea-registry
volumes:
- name: trade-api-wrapper
configMap:
name: trade-api-wrapper
- name: trade-api-upstream
configMap:
name: trade-api-upstream
containers:
- name: api
image: rv32i.pl/trade/trade-api:k3s-20260106013603
image: gitea.mpabi.pl/trade/trade-api:k3s-20260111095435
imagePullPolicy: IfNotPresent
command: ["node", "/override/wrapper.mjs"]
ports:
- name: http
containerPort: 8787
volumeMounts:
- name: trade-api-wrapper
mountPath: /override/wrapper.mjs
subPath: wrapper.mjs
readOnly: true
- name: trade-api-upstream
mountPath: /app/services/api/server.mjs
subPath: server.mjs
readOnly: true
env:
- name: PORT
value: "8787"
- name: UPSTREAM_PORT
value: "8788"
- name: UPSTREAM_ENTRY
value: "/app/services/api/server.mjs"
- name: UPSTREAM_HOST
value: "127.0.0.1"
- name: APP_VERSION
value: "staging"
- name: HASURA_GRAPHQL_URL

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,783 @@
import crypto from 'node:crypto';
import http from 'node:http';
import { spawn } from 'node:child_process';
const WRAPPER_PORT = Number.parseInt(String(process.env.PORT || '8787'), 10);
const UPSTREAM_PORT = Number.parseInt(String(process.env.UPSTREAM_PORT || '8788'), 10);
const UPSTREAM_HOST = String(process.env.UPSTREAM_HOST || '127.0.0.1');
const UPSTREAM_ENTRY = String(process.env.UPSTREAM_ENTRY || '/app/services/api/server.mjs');
const HASURA_URL = String(process.env.HASURA_GRAPHQL_URL || 'http://hasura:8080/v1/graphql');
const HASURA_ADMIN_SECRET = String(process.env.HASURA_ADMIN_SECRET || '');
const CORS_ORIGIN = String(process.env.CORS_ORIGIN || '*');
if (!Number.isInteger(WRAPPER_PORT) || WRAPPER_PORT <= 0) throw new Error('Invalid PORT');
if (!Number.isInteger(UPSTREAM_PORT) || UPSTREAM_PORT <= 0) throw new Error('Invalid UPSTREAM_PORT');
function getIsoNow() {
return new Date().toISOString();
}
function sha256Hex(text) {
return crypto.createHash('sha256').update(text, 'utf8').digest('hex');
}
function getHeader(req, name) {
const v = req.headers[String(name).toLowerCase()];
return Array.isArray(v) ? v[0] : v;
}
function readBearerToken(req) {
const auth = getHeader(req, 'authorization');
if (auth && typeof auth === 'string') {
const m = auth.match(/^Bearer\s+(.+)$/i);
if (m && m[1]) return m[1].trim();
}
const apiKey = getHeader(req, 'x-api-key');
if (apiKey && typeof apiKey === 'string' && apiKey.trim()) return apiKey.trim();
return undefined;
}
function withCors(res) {
res.setHeader('access-control-allow-origin', CORS_ORIGIN);
res.setHeader('access-control-allow-methods', 'GET,POST,OPTIONS');
res.setHeader(
'access-control-allow-headers',
'content-type, authorization, x-api-key, x-admin-secret'
);
}
function sendJson(res, status, body) {
withCors(res);
res.statusCode = status;
res.setHeader('content-type', 'application/json; charset=utf-8');
res.end(JSON.stringify(body));
}
async function readBodyJson(req, { maxBytes }) {
const chunks = [];
let total = 0;
for await (const chunk of req) {
total += chunk.length;
if (total > maxBytes) throw new Error('payload_too_large');
chunks.push(chunk);
}
const text = Buffer.concat(chunks).toString('utf8');
if (!text.trim()) return {};
try {
return JSON.parse(text);
} catch {
throw new Error('invalid_json');
}
}
function parseNumeric(value) {
if (value == null) return null;
if (typeof value === 'number') return Number.isFinite(value) ? value : null;
if (typeof value === 'string') {
const s = value.trim();
if (!s) return null;
const n = Number(s);
return Number.isFinite(n) ? n : null;
}
return null;
}
function clampInt(value, min, max) {
const n = Number.parseInt(String(value), 10);
if (!Number.isFinite(n) || !Number.isInteger(n)) return min;
return Math.min(max, Math.max(min, n));
}
function clampNumber(value, min, max, fallback) {
const n = typeof value === 'number' ? value : Number(value);
if (!Number.isFinite(n)) return fallback;
return Math.min(max, Math.max(min, n));
}
function isUuid(value) {
const s = String(value || '').trim();
return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(s);
}
async function hasuraRequest(query, variables) {
if (!HASURA_ADMIN_SECRET) throw new Error('Missing HASURA_ADMIN_SECRET');
const res = await fetch(HASURA_URL, {
method: 'POST',
headers: {
'content-type': 'application/json',
'x-hasura-admin-secret': HASURA_ADMIN_SECRET,
},
body: JSON.stringify({ query, variables }),
});
const text = await res.text();
if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`);
const json = text ? JSON.parse(text) : {};
if (json.errors?.length) throw new Error(json.errors.map((e) => e.message).join(' | '));
return json.data;
}
function normalizeScopes(value) {
if (!value) return [];
if (Array.isArray(value)) return value.map((v) => String(v)).filter(Boolean);
if (typeof value === 'string') {
return value
.split(',')
.map((s) => s.trim())
.filter(Boolean);
}
return [];
}
async function requireValidToken(req, requiredScope) {
const token = readBearerToken(req);
if (!token) return { ok: false, status: 401, error: 'missing_token' };
const hash = sha256Hex(token);
const query = `
query ValidToken($hash: String!) {
api_tokens(where: {token_hash: {_eq: $hash}, revoked_at: {_is_null: true}}, limit: 1) {
id
name
scopes
}
}
`;
let data;
try {
data = await hasuraRequest(query, { hash });
} catch (err) {
return { ok: false, status: 500, error: String(err?.message || err) };
}
const row = data?.api_tokens?.[0];
if (!row?.id) return { ok: false, status: 401, error: 'invalid_or_revoked_token' };
const scopes = normalizeScopes(row.scopes);
if (requiredScope && !scopes.includes(requiredScope)) {
return { ok: false, status: 403, error: 'missing_scope' };
}
// best-effort touch
const touch = `
mutation TouchToken($id: uuid!, $ts: timestamptz!) {
update_api_tokens_by_pk(pk_columns: {id: $id}, _set: {last_used_at: $ts}) { id }
}
`;
hasuraRequest(touch, { id: row.id, ts: getIsoNow() }).catch(() => {});
return { ok: true, token: { id: row.id, name: row.name } };
}
async function readSolPriceUsd() {
try {
const q = `
query SolPrice {
dlob_stats_latest(where: {market_name: {_eq: "SOL-PERP"}}, limit: 1) {
mid_price
mark_price
oracle_price
}
}
`;
const data = await hasuraRequest(q, {});
const row = data?.dlob_stats_latest?.[0];
const p = parseNumeric(row?.mid_price) ?? parseNumeric(row?.mark_price) ?? parseNumeric(row?.oracle_price);
if (p != null && p > 0) return p;
} catch {
// ignore
}
return null;
}
function getByPath(obj, pathStr) {
if (!obj || typeof obj !== 'object') return undefined;
const parts = String(pathStr || '').split('.').filter(Boolean);
let cur = obj;
for (const p of parts) {
if (!cur || typeof cur !== 'object') return undefined;
cur = cur[p];
}
return cur;
}
function readNumberFromPayload(payload, paths) {
for (const p of paths) {
const v = getByPath(payload, p);
const n = parseNumeric(v);
if (n != null) return n;
}
return null;
}
function readTextFromPayload(payload, paths) {
for (const p of paths) {
const v = getByPath(payload, p);
if (typeof v === 'string' && v.trim()) return v.trim();
}
return null;
}
function inferContractSizeUsd(contract) {
return (
readNumberFromPayload(contract, [
'desired.size_usd',
'desired.sizeUsd',
'desired.notional_usd',
'desired.notionalUsd',
'entry.size_usd',
'entry.sizeUsd',
'entry.notional_usd',
'entry.notionalUsd',
'entry.order_intent.size_usd',
'entry.order_intent.sizeUsd',
'desired.order_intent.size_usd',
'desired.order_intent.sizeUsd',
]) || null
);
}
function inferContractSide(contract) {
const raw =
readTextFromPayload(contract, [
'desired.side',
'entry.side',
'entry.order_intent.side',
'desired.order_intent.side',
'desired.direction',
'entry.direction',
]) || '';
const v = raw.toLowerCase();
if (v === 'long' || v === 'buy') return 'long';
if (v === 'short' || v === 'sell') return 'short';
return null;
}
function sumCostsFromEvents(events) {
const totals = {
tradeFeeUsd: 0,
txFeeUsd: 0,
slippageUsd: 0,
fundingUsd: 0,
realizedPnlUsd: 0,
txCount: 0,
fillCount: 0,
cancelCount: 0,
modifyCount: 0,
errorCount: 0,
};
for (const ev of events || []) {
const t = String(ev?.event_type || '').toLowerCase();
const payload = ev?.payload && typeof ev.payload === 'object' ? ev.payload : {};
const tradeFeeUsd =
readNumberFromPayload(payload, [
'realized_fee_usd',
'trade_fee_usd',
'fee_usd',
'fees.trade_fee_usd',
'fees.usd',
]) || 0;
const txFeeUsd =
readNumberFromPayload(payload, [
'realized_tx_usd',
'tx_fee_usd',
'network_fee_usd',
'fees.tx_fee_usd',
'fees.network_usd',
]) || 0;
const slippageUsd =
readNumberFromPayload(payload, [
'slippage_usd',
'realized_slippage_usd',
'execution_usd',
'realized_execution_usd',
]) || 0;
const fundingUsd = readNumberFromPayload(payload, ['funding_usd', 'realized_funding_usd']) || 0;
const pnlUsd = readNumberFromPayload(payload, ['realized_pnl_usd', 'pnl_usd']) || 0;
const txCount = readNumberFromPayload(payload, ['tx_count', 'txCount']) || 0;
totals.tradeFeeUsd += tradeFeeUsd;
totals.txFeeUsd += txFeeUsd;
totals.slippageUsd += slippageUsd;
totals.fundingUsd += fundingUsd;
totals.realizedPnlUsd += pnlUsd;
totals.txCount += txCount;
if (t.includes('fill')) totals.fillCount += 1;
if (t.includes('cancel')) totals.cancelCount += 1;
if (t.includes('modify') || t.includes('reprice')) totals.modifyCount += 1;
if (t.includes('error') || String(ev?.severity || '').toLowerCase() === 'error') totals.errorCount += 1;
}
const totalCostsUsd = totals.tradeFeeUsd + totals.txFeeUsd + totals.slippageUsd + totals.fundingUsd;
return {
...totals,
totalCostsUsd,
netPnlUsd: totals.realizedPnlUsd - totalCostsUsd,
};
}
function buildCostSeriesFromEvents(events, { maxPoints }) {
const points = [];
const totals = {
tradeFeeUsd: 0,
txFeeUsd: 0,
slippageUsd: 0,
fundingUsd: 0,
realizedPnlUsd: 0,
};
for (const ev of events || []) {
const ts = ev?.ts;
if (!ts) continue;
const payload = ev?.payload && typeof ev.payload === 'object' ? ev.payload : {};
const tradeFeeUsd =
readNumberFromPayload(payload, [
'realized_fee_usd',
'trade_fee_usd',
'fee_usd',
'fees.trade_fee_usd',
'fees.usd',
]) || 0;
const txFeeUsd =
readNumberFromPayload(payload, [
'realized_tx_usd',
'tx_fee_usd',
'network_fee_usd',
'fees.tx_fee_usd',
'fees.network_usd',
]) || 0;
const slippageUsd =
readNumberFromPayload(payload, [
'slippage_usd',
'realized_slippage_usd',
'execution_usd',
'realized_execution_usd',
]) || 0;
const fundingUsd = readNumberFromPayload(payload, ['funding_usd', 'realized_funding_usd']) || 0;
const pnlUsd = readNumberFromPayload(payload, ['realized_pnl_usd', 'pnl_usd']) || 0;
totals.tradeFeeUsd += tradeFeeUsd;
totals.txFeeUsd += txFeeUsd;
totals.slippageUsd += slippageUsd;
totals.fundingUsd += fundingUsd;
totals.realizedPnlUsd += pnlUsd;
const totalCostsUsd = totals.tradeFeeUsd + totals.txFeeUsd + totals.slippageUsd + totals.fundingUsd;
points.push({
ts,
tradeFeeUsd: totals.tradeFeeUsd,
txFeeUsd: totals.txFeeUsd,
slippageUsd: totals.slippageUsd,
fundingUsd: totals.fundingUsd,
totalCostsUsd,
realizedPnlUsd: totals.realizedPnlUsd,
netPnlUsd: totals.realizedPnlUsd - totalCostsUsd,
});
}
const cap = Math.max(50, Math.min(10_000, Number(maxPoints) || 600));
if (points.length <= cap) return points;
const step = Math.ceil(points.length / cap);
const sampled = [];
for (let i = 0; i < points.length; i += step) sampled.push(points[i]);
const last = points[points.length - 1];
if (sampled[sampled.length - 1] !== last) sampled.push(last);
return sampled;
}
function proxyToUpstream(req, res) {
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
const headers = { ...req.headers };
delete headers.host;
delete headers.connection;
const opts = {
host: UPSTREAM_HOST,
port: UPSTREAM_PORT,
method: req.method,
path: url.pathname + url.search,
headers,
};
const upstreamReq = http.request(opts, (upstreamRes) => {
withCors(res);
res.statusCode = upstreamRes.statusCode || 502;
for (const [k, v] of Object.entries(upstreamRes.headers || {})) {
if (!k) continue;
if (k.toLowerCase() === 'content-length') continue;
if (k.toLowerCase().startsWith('access-control-')) continue;
if (v != null) res.setHeader(k, v);
}
upstreamRes.pipe(res);
});
upstreamReq.on('error', (err) => {
sendJson(res, 502, { ok: false, error: String(err?.message || err) });
});
req.pipe(upstreamReq);
}
async function handleMonitor(req, res, url) {
const auth = await requireValidToken(req, 'read');
if (!auth.ok) {
sendJson(res, auth.status, { ok: false, error: auth.error });
return;
}
const pathname = url.pathname;
const parts = pathname.split('/').filter(Boolean);
const contractId = parts[2];
if (!isUuid(contractId)) {
sendJson(res, 400, { ok: false, error: 'invalid_contract_id' });
return;
}
const limit = clampInt(url.searchParams.get('eventsLimit') || '2000', 10, 50_000);
const wantSeries = (url.searchParams.get('series') || '').trim() === '1';
const seriesMax = clampInt(url.searchParams.get('seriesMax') || '600', 50, 10_000);
const qContract = `
query ContractByPk($id: uuid!) {
bot_contracts_by_pk(id: $id) {
id
decision_id
bot_id
model_version
market_name
subaccount_id
status
desired
entry
manage
exit
gates
created_at
updated_at
last_heartbeat_at
ended_at
reason
}
}
`;
const qEvents = `
query ContractEvents($id: uuid!, $limit: Int!) {
bot_events(where: {contract_id: {_eq: $id}}, order_by: {ts: asc}, limit: $limit) {
ts
contract_id
decision_id
bot_id
market_name
event_type
severity
payload
}
}
`;
try {
const data = await hasuraRequest(qContract, { id: contractId });
const contract = data?.bot_contracts_by_pk;
if (!contract?.id) {
sendJson(res, 404, { ok: false, error: 'contract_not_found' });
return;
}
const evData = await hasuraRequest(qEvents, { id: contractId, limit });
const events = evData?.bot_events || [];
const costs = sumCostsFromEvents(events);
const series = wantSeries ? buildCostSeriesFromEvents(events, { maxPoints: seriesMax }) : null;
const sizeUsd = inferContractSizeUsd(contract);
const side = inferContractSide(contract);
let closeEst = null;
if (contract.market_name && sizeUsd != null) {
const qSlip = `
query Slippage($market: String!) {
dlob_slippage_latest(where: {market_name: {_eq: $market}}) {
market_name
side
size_usd
mid_price
vwap_price
worst_price
impact_bps
fill_pct
updated_at
}
}
`;
const slipData = await hasuraRequest(qSlip, { market: contract.market_name });
const rows = slipData?.dlob_slippage_latest || [];
const pickNearest = (wantedSide) => {
const candidates = rows.filter((r) => String(r.side || '').toLowerCase() === wantedSide);
if (!candidates.length) return null;
let best = null;
let bestD = Infinity;
for (const r of candidates) {
const s = parseNumeric(r.size_usd);
if (s == null) continue;
const d = Math.abs(s - sizeUsd);
if (d < bestD) {
bestD = d;
best = r;
}
}
return best;
};
closeEst = {
requestedSizeUsd: sizeUsd,
entrySide: side,
suggestedCloseSide: side === 'long' ? 'sell' : side === 'short' ? 'buy' : null,
buy: pickNearest('buy'),
sell: pickNearest('sell'),
};
}
sendJson(res, 200, {
ok: true,
contract,
eventsCount: events.length,
costs,
series,
closeEstimate: closeEst,
});
} catch (err) {
sendJson(res, 500, { ok: false, error: String(err?.message || err) });
}
}
async function handleEstimate(req, res) {
const auth = await requireValidToken(req, 'read');
if (!auth.ok) {
sendJson(res, auth.status, { ok: false, error: auth.error });
return;
}
let body;
try {
body = await readBodyJson(req, { maxBytes: 256 * 1024 });
} catch (err) {
const msg = String(err?.message || err);
if (msg === 'payload_too_large') {
sendJson(res, 413, { ok: false, error: 'payload_too_large' });
return;
}
sendJson(res, 400, { ok: false, error: 'invalid_json' });
return;
}
const market = String(body?.market_name || body?.market || '').trim();
if (!market) {
sendJson(res, 400, { ok: false, error: 'missing_market' });
return;
}
const notionalUsd = parseNumeric(body?.notional_usd ?? body?.notionalUsd ?? body?.size_usd ?? body?.sizeUsd);
if (!(notionalUsd != null && notionalUsd > 0)) {
sendJson(res, 400, { ok: false, error: 'invalid_notional_usd' });
return;
}
const entrySideRaw = String(body?.side || 'long').trim().toLowerCase();
const entrySide = entrySideRaw === 'short' ? 'short' : 'long';
const orderType = String(body?.order_type || body?.orderType || 'market').trim().toLowerCase();
const isMarket = orderType === 'market' || orderType === 'taker';
const takerBps = clampNumber(parseNumeric(body?.fee_taker_bps) ?? 5, 0, 1000, 5);
const makerBps = clampNumber(parseNumeric(body?.fee_maker_bps) ?? 0, -1000, 1000, 0);
const feeBps = isMarket ? takerBps : makerBps;
let txFeeUsdEst = parseNumeric(body?.tx_fee_usd_est);
if (txFeeUsdEst == null) {
const baseLamports = 5000;
const sigs = 1;
const priorityLamports = 0;
const lamports = baseLamports * sigs + priorityLamports;
const sol = lamports / 1_000_000_000;
const solUsd = await readSolPriceUsd();
txFeeUsdEst = solUsd != null ? sol * solUsd : 0;
}
txFeeUsdEst = clampNumber(txFeeUsdEst, 0, 100, 0);
const expectedReprices = clampInt(body?.expected_reprices_per_entry ?? body?.expectedReprices ?? '0', 0, 500);
const modifyTxCount = clampInt(body?.modify_tx_count ?? body?.modifyTxCount ?? '2', 0, 10);
try {
const qSlip = `
query Slippage($market: String!) {
dlob_slippage_latest(where: {market_name: {_eq: $market}}) {
market_name
side
size_usd
mid_price
vwap_price
worst_price
impact_bps
fill_pct
updated_at
}
}
`;
const slipData = await hasuraRequest(qSlip, { market });
const rows = slipData?.dlob_slippage_latest || [];
const wantedSide = entrySide === 'long' ? 'buy' : 'sell';
const candidates = rows.filter((r) => String(r.side || '').toLowerCase() === wantedSide);
let best = null;
let bestD = Infinity;
for (const r of candidates) {
const s = parseNumeric(r.size_usd);
if (s == null) continue;
const d = Math.abs(s - notionalUsd);
if (d < bestD) {
bestD = d;
best = r;
}
}
const impactBps = parseNumeric(best?.impact_bps) ?? 0;
const slippageUsd = (notionalUsd * impactBps) / 10_000;
const tradeFeeUsd = (notionalUsd * feeBps) / 10_000;
const modifyCostUsd = expectedReprices * modifyTxCount * txFeeUsdEst;
const totalUsd = tradeFeeUsd + slippageUsd + txFeeUsdEst + modifyCostUsd;
const totalBps = (totalUsd / notionalUsd) * 10_000;
sendJson(res, 200, {
ok: true,
input: {
market_name: market,
notional_usd: notionalUsd,
side: entrySide,
order_type: orderType,
fee_bps: feeBps,
tx_fee_usd_est: txFeeUsdEst,
expected_reprices_per_entry: expectedReprices,
},
dlob: best
? {
size_usd: best.size_usd,
side: best.side,
mid_price: best.mid_price,
vwap_price: best.vwap_price,
impact_bps: best.impact_bps,
fill_pct: best.fill_pct,
updated_at: best.updated_at,
}
: null,
breakdown: {
trade_fee_usd: tradeFeeUsd,
slippage_usd: slippageUsd,
tx_fee_usd: txFeeUsdEst,
expected_modify_usd: modifyCostUsd,
total_usd: totalUsd,
total_bps: totalBps,
breakeven_bps: totalBps,
},
});
} catch (err) {
sendJson(res, 500, { ok: false, error: String(err?.message || err) });
}
}
let upstreamChild = null;
function startUpstream() {
const env = { ...process.env, PORT: String(UPSTREAM_PORT) };
upstreamChild = spawn('node', [UPSTREAM_ENTRY], { env, stdio: 'inherit' });
upstreamChild.on('exit', (code, signal) => {
console.error(`upstream exited: code=${code} signal=${signal}`);
});
}
function shutdown() {
if (upstreamChild && !upstreamChild.killed) upstreamChild.kill('SIGTERM');
}
process.on('SIGTERM', shutdown);
process.on('SIGINT', shutdown);
startUpstream();
const server = http.createServer(async (req, res) => {
if (req.method === 'OPTIONS') {
withCors(res);
res.statusCode = 204;
res.end();
return;
}
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
const pathname = url.pathname;
if (req.method === 'GET' && pathname === '/healthz') {
// Check upstream quickly; if it's down, we fail readiness.
const opts = { host: UPSTREAM_HOST, port: UPSTREAM_PORT, path: '/healthz', method: 'GET', timeout: 800 };
const upstreamOk = await new Promise((resolve) => {
const r = http.request(opts, (rr) => {
rr.resume();
resolve(rr.statusCode === 200);
});
r.on('timeout', () => {
r.destroy();
resolve(false);
});
r.on('error', () => resolve(false));
r.end();
});
if (!upstreamOk) {
sendJson(res, 503, { ok: false, error: 'upstream_not_ready' });
return;
}
sendJson(res, 200, {
ok: true,
service: 'trade-api-wrapper',
startedAt: getIsoNow(),
upstream: { host: UPSTREAM_HOST, port: UPSTREAM_PORT },
});
return;
}
if (req.method === 'POST' && pathname === '/v1/contracts/costs/estimate') {
await handleEstimate(req, res);
return;
}
if (req.method === 'GET' && pathname.startsWith('/v1/contracts/') && pathname.endsWith('/monitor')) {
await handleMonitor(req, res, url);
return;
}
proxyToUpstream(req, res);
});
server.listen(WRAPPER_PORT, () => {
console.log(
JSON.stringify(
{
service: 'trade-api-wrapper',
port: WRAPPER_PORT,
upstream: { entry: UPSTREAM_ENTRY, host: UPSTREAM_HOST, port: UPSTREAM_PORT },
hasuraUrl: HASURA_URL,
hasuraAdminSecret: HASURA_ADMIN_SECRET ? '***' : undefined,
},
null,
2
)
);
});

View File

@@ -0,0 +1,48 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: candles-cache-worker
annotations:
argocd.argoproj.io/sync-wave: "6"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: candles-cache-worker
template:
metadata:
labels:
app.kubernetes.io/name: candles-cache-worker
spec:
containers:
- name: worker
image: node:20-slim
imagePullPolicy: IfNotPresent
env:
- name: HASURA_GRAPHQL_URL
value: http://hasura:8080/v1/graphql
- name: HASURA_ADMIN_SECRET
valueFrom:
secretKeyRef:
name: trade-hasura
key: HASURA_GRAPHQL_ADMIN_SECRET
- name: CANDLES_SYMBOLS
value: SOL-PERP,PUMP-PERP
- name: CANDLES_SOURCES
value: any
- name: CANDLES_TFS
value: 1s,3s,5s,15s,30s,1m,3m,5m,15m,30m,1h,4h,12h,1d
- name: CANDLES_TARGET_POINTS
value: "1024"
- name: CANDLES_POLL_MS
value: "5000"
command: ["node", "/app/worker.mjs"]
volumeMounts:
- name: script
mountPath: /app/worker.mjs
subPath: worker.mjs
readOnly: true
volumes:
- name: script
configMap:
name: candles-cache-worker-script

View File

@@ -0,0 +1,431 @@
import fs from 'node:fs';
import process from 'node:process';
import { setTimeout as sleep } from 'node:timers/promises';
function readJsonFile(filePath) {
try {
const raw = fs.readFileSync(filePath, 'utf8');
return JSON.parse(raw);
} catch {
return undefined;
}
}
function getIsoNow() {
return new Date().toISOString();
}
function clampInt(value, min, max, fallback) {
const n = Number.parseInt(String(value ?? ''), 10);
if (!Number.isInteger(n)) return fallback;
return Math.min(max, Math.max(min, n));
}
function envList(name, fallbackCsv) {
const raw = process.env[name] ?? fallbackCsv;
return String(raw)
.split(',')
.map((s) => s.trim())
.filter(Boolean);
}
function parseTimeframeToSeconds(tf) {
const v = String(tf || '').trim().toLowerCase();
if (!v) return 60;
const m = v.match(/^(\d+)(s|m|h|d)$/);
if (!m) throw new Error('invalid_tf');
const num = Number.parseInt(m[1], 10);
if (!Number.isInteger(num) || num <= 0) throw new Error('invalid_tf');
const unit = m[2];
const mult = unit === 's' ? 1 : unit === 'm' ? 60 : unit === 'h' ? 3600 : 86400;
return num * mult;
}
function sqlLit(value) {
const s = String(value ?? '');
return `'${s.replace(/'/g, "''")}'`;
}
function resolveHasuraBaseUrl(graphqlUrlOrBase) {
const u = String(graphqlUrlOrBase || '').trim();
if (!u) return 'http://hasura:8080';
// common case: http://hasura:8080/v1/graphql
if (u.endsWith('/v1/graphql')) return u.slice(0, -'/v1/graphql'.length);
return u.replace(/\/$/, '');
}
function resolveConfig() {
const tokensPath =
process.env.HASURA_TOKENS_FILE ||
process.env.TOKENS_FILE ||
process.env.HASURA_CONFIG_FILE ||
'/app/tokens/hasura.json';
const tokens = readJsonFile(tokensPath) || {};
const graphqlUrl =
process.env.HASURA_GRAPHQL_URL ||
tokens.graphqlUrl ||
tokens.apiUrl ||
'http://hasura:8080/v1/graphql';
const hasuraBaseUrl = resolveHasuraBaseUrl(graphqlUrl);
const hasuraAdminSecret =
process.env.HASURA_ADMIN_SECRET ||
process.env.HASURA_GRAPHQL_ADMIN_SECRET ||
tokens.adminSecret ||
tokens.hasuraAdminSecret;
if (!hasuraAdminSecret) throw new Error('Missing HASURA_ADMIN_SECRET (required for /v2/query run_sql)');
const symbols = envList('CANDLES_SYMBOLS', 'SOL-PERP,PUMP-PERP');
// sources: include "any" (sourceKey='') by default.
const rawSources = envList('CANDLES_SOURCES', 'any');
const sources = [];
for (const s of rawSources) {
const v = String(s).trim();
if (!v) continue;
if (v.toLowerCase() === 'any') sources.push('');
else sources.push(v);
}
if (!sources.includes('')) sources.unshift('');
const tfList = envList('CANDLES_TFS', '1s,3s,5s,15s,30s,1m,3m,5m,15m,30m,1h,4h,12h,1d');
const bucketSecondsList = tfList
.map((t) => {
try {
return parseTimeframeToSeconds(t);
} catch {
return null;
}
})
.filter((n) => n != null)
.map((n) => n)
.filter((n, idx, arr) => arr.findIndex((x) => x === n) === idx)
.sort((a, b) => a - b);
const targetPoints = clampInt(process.env.CANDLES_TARGET_POINTS, 10, 100_000, 1024);
// legacy: kept for compatibility; if set, used as a minimum warmup window (days).
const backfillDays = clampInt(process.env.CANDLES_BACKFILL_DAYS, 0, 3650, 0);
const pollMs = clampInt(process.env.CANDLES_POLL_MS, 250, 60_000, 5000);
return { hasuraBaseUrl, hasuraAdminSecret, symbols, sources, bucketSecondsList, targetPoints, backfillDays, pollMs };
}
async function hasuraRunSql(cfg, sql, { readOnly } = { readOnly: false }) {
const url = `${cfg.hasuraBaseUrl}/v2/query`;
const body = {
type: 'run_sql',
args: {
source: 'default',
sql,
read_only: Boolean(readOnly),
},
};
const res = await fetch(url, {
method: 'POST',
headers: { 'content-type': 'application/json', 'x-hasura-admin-secret': cfg.hasuraAdminSecret },
body: JSON.stringify(body),
signal: AbortSignal.timeout(60_000),
});
const text = await res.text();
if (!res.ok) throw new Error(`Hasura run_sql HTTP ${res.status}: ${text}`);
return JSON.parse(text);
}
function chunkSecondsForBucket(bucketSeconds) {
if (bucketSeconds <= 5) return 15 * 60;
if (bucketSeconds <= 60) return 60 * 60;
if (bucketSeconds <= 300) return 6 * 60 * 60;
if (bucketSeconds <= 3600) return 24 * 60 * 60;
return 7 * 24 * 60 * 60;
}
function sqlUpsertCandlesFromTicks({ symbol, sourceKey, bucketSeconds, fromIso, toIso }) {
return `
WITH chosen AS (
SELECT source AS chosen_source
FROM public.drift_ticks
WHERE symbol = ${sqlLit(symbol)}
ORDER BY ts DESC
LIMIT 1
),
base AS (
SELECT
time_bucket(make_interval(secs => ${bucketSeconds}), ts) AS bucket,
ts,
COALESCE(mark_price, oracle_price) AS px,
oracle_price AS oracle_px
FROM public.drift_ticks
WHERE symbol = ${sqlLit(symbol)}
AND ts >= ${sqlLit(fromIso)}::timestamptz
AND ts < ${sqlLit(toIso)}::timestamptz
AND (
${sqlLit(sourceKey)} <> '' AND source = ${sqlLit(sourceKey)}
OR ${sqlLit(sourceKey)} = '' AND source = COALESCE((SELECT chosen_source FROM chosen), source)
)
),
agg AS (
SELECT
bucket,
(array_agg(px ORDER BY ts ASC))[1] AS open,
max(px) AS high,
min(px) AS low,
(array_agg(px ORDER BY ts DESC))[1] AS close,
(array_agg(oracle_px ORDER BY ts ASC))[1] AS oracle_open,
max(oracle_px) AS oracle_high,
min(oracle_px) AS oracle_low,
(array_agg(oracle_px ORDER BY ts DESC))[1] AS oracle_close,
count(*)::bigint AS ticks
FROM base
GROUP BY bucket
)
INSERT INTO public.drift_candles_cache
(
bucket,
bucket_seconds,
symbol,
source,
open,
high,
low,
close,
oracle_open,
oracle_high,
oracle_low,
oracle_close,
ticks,
updated_at
)
SELECT
bucket,
${bucketSeconds},
${sqlLit(symbol)},
${sqlLit(sourceKey)},
open,
high,
low,
close,
oracle_open,
oracle_high,
oracle_low,
oracle_close,
ticks,
now()
FROM agg
ON CONFLICT (bucket, bucket_seconds, symbol, source) DO UPDATE SET
open = EXCLUDED.open,
high = EXCLUDED.high,
low = EXCLUDED.low,
close = EXCLUDED.close,
oracle_open = EXCLUDED.oracle_open,
oracle_high = EXCLUDED.oracle_high,
oracle_low = EXCLUDED.oracle_low,
oracle_close = EXCLUDED.oracle_close,
ticks = EXCLUDED.ticks,
updated_at = now();
`;
}
function sqlDeleteOlderCandles({ symbol, sourceKey, bucketSeconds, cutoffIso }) {
return `
DELETE FROM public.drift_candles_cache
WHERE symbol = ${sqlLit(symbol)}
AND source = ${sqlLit(sourceKey)}
AND bucket_seconds = ${bucketSeconds}
AND bucket < ${sqlLit(cutoffIso)}::timestamptz;
`;
}
async function getTickRange(cfg, { symbol, sourceKey }) {
const sql =
String(sourceKey) === ''
? `
WITH chosen AS (
SELECT source AS chosen_source
FROM public.drift_ticks
WHERE symbol=${sqlLit(symbol)}
ORDER BY ts DESC
LIMIT 1
)
SELECT min(ts) AS min_ts, max(ts) AS max_ts
FROM public.drift_ticks
WHERE symbol=${sqlLit(symbol)}
AND source = COALESCE((SELECT chosen_source FROM chosen), source);
`
: `
SELECT min(ts) AS min_ts, max(ts) AS max_ts
FROM public.drift_ticks
WHERE symbol=${sqlLit(symbol)}
AND source = ${sqlLit(sourceKey)};
`;
const out = await hasuraRunSql(cfg, sql, { readOnly: true });
const row = Array.isArray(out?.result) && out.result.length >= 2 ? out.result[1] : null;
if (!row) return { minTs: null, maxTs: null };
const minTs = row[0] ? String(row[0]).trim() : null;
const maxTs = row[1] ? String(row[1]).trim() : null;
return { minTs: minTs && minTs.length ? minTs : null, maxTs: maxTs && maxTs.length ? maxTs : null };
}
function desiredFromIso({ minTsIso, maxTsIso, bucketSeconds, targetPoints, backfillDays }) {
const endMs = Date.parse(maxTsIso);
const minMs = minTsIso ? Date.parse(minTsIso) : null;
const wantSpanMs = targetPoints * bucketSeconds * 1000;
const wantFromMs = endMs - wantSpanMs;
const minFromMs = backfillDays > 0 ? endMs - backfillDays * 24 * 60 * 60 * 1000 : null;
let fromMs = wantFromMs;
if (minFromMs != null) fromMs = Math.min(fromMs, minFromMs);
if (minMs != null) fromMs = Math.max(fromMs, minMs);
return new Date(Math.max(0, fromMs)).toISOString();
}
function safetyWindowSeconds(bucketSeconds) {
if (bucketSeconds <= 60) return 10 * 60;
if (bucketSeconds <= 300) return 60 * 60;
if (bucketSeconds <= 3600) return 6 * 60 * 60;
if (bucketSeconds <= 14_400) return 24 * 60 * 60;
return 2 * 24 * 60 * 60;
}
async function backfillBucket(cfg, { symbol, sourceKey, bucketSeconds, fromIso, toIso }) {
const chunk = chunkSecondsForBucket(bucketSeconds);
for (let t = Date.parse(fromIso); t < Date.parse(toIso); t += chunk * 1000) {
const a = new Date(t).toISOString();
const b = new Date(Math.min(Date.parse(toIso), t + chunk * 1000)).toISOString();
await hasuraRunSql(cfg, sqlUpsertCandlesFromTicks({ symbol, sourceKey, bucketSeconds, fromIso: a, toIso: b }));
}
}
async function getMaxBucket(cfg, { symbol, sourceKey, bucketSeconds }) {
const sql = `
SELECT max(bucket) AS max_bucket
FROM public.drift_candles_cache
WHERE symbol=${sqlLit(symbol)} AND source=${sqlLit(sourceKey)} AND bucket_seconds=${bucketSeconds};
`;
const out = await hasuraRunSql(cfg, sql, { readOnly: true });
// Hasura returns {result_type, result:[[col...],[row...]]}
const row = Array.isArray(out?.result) && out.result.length >= 2 ? out.result[1] : null;
const v = row && row[0] ? String(row[0]) : null;
return v && v.trim() ? v.trim() : null;
}
async function main() {
const cfg = resolveConfig();
console.log(
JSON.stringify(
{
service: 'candles-cache-worker',
startedAt: getIsoNow(),
hasuraBaseUrl: cfg.hasuraBaseUrl,
symbols: cfg.symbols,
sources: cfg.sources.map((s) => (s ? s : '(any)')),
bucketSecondsList: cfg.bucketSecondsList,
targetPoints: cfg.targetPoints,
backfillDays: cfg.backfillDays,
pollMs: cfg.pollMs,
},
null,
2
)
);
// Backfill to warm cache: for each timeframe keep ~targetPoints candles (or "as much as we have").
for (const symbol of cfg.symbols) {
for (const sourceKey of cfg.sources) {
const range = await getTickRange(cfg, { symbol, sourceKey });
if (!range.maxTs) continue;
const toIso = new Date(Date.parse(range.maxTs)).toISOString();
for (const bs of cfg.bucketSecondsList) {
const fromIso = desiredFromIso({
minTsIso: range.minTs,
maxTsIso: toIso,
bucketSeconds: bs,
targetPoints: cfg.targetPoints,
backfillDays: cfg.backfillDays,
});
console.log(
`[candles-cache-worker] warmup symbol=${symbol} source=${sourceKey || '(any)'} bs=${bs}s from=${fromIso} to=${toIso} points=${cfg.targetPoints}`
);
try {
await backfillBucket(cfg, { symbol, sourceKey, bucketSeconds: bs, fromIso, toIso });
// Enforce max window for this bucket (derived data; safe to prune).
await hasuraRunSql(cfg, sqlDeleteOlderCandles({ symbol, sourceKey, bucketSeconds: bs, cutoffIso: fromIso }));
} catch (err) {
console.error(
`[candles-cache-worker] warmup failed (${symbol}/${sourceKey || 'any'}/${bs}s): ${String(err?.message || err)}`
);
}
}
}
}
// Prime last buckets.
const last = new Map(); // key -> iso bucket
for (const symbol of cfg.symbols) {
for (const sourceKey of cfg.sources) {
for (const bs of cfg.bucketSecondsList) {
const k = `${symbol}::${sourceKey}::${bs}`;
try {
const maxBucket = await getMaxBucket(cfg, { symbol, sourceKey, bucketSeconds: bs });
if (maxBucket) last.set(k, maxBucket);
} catch {
// ignore
}
}
}
}
while (true) {
const loopNow = Date.now();
const loopIso = new Date(loopNow).toISOString();
const pruneEveryMs = 60_000;
const lastPruneAt = last.__pruneAt || new Map();
// stash on the Map to keep closure-local without introducing a new outer var
last.__pruneAt = lastPruneAt;
for (const symbol of cfg.symbols) {
for (const sourceKey of cfg.sources) {
for (const bs of cfg.bucketSecondsList) {
const k = `${symbol}::${sourceKey}::${bs}`;
const prev = last.get(k);
const safety = safetyWindowSeconds(bs);
const fromMs = prev ? Date.parse(prev) - safety * 1000 : loopNow - safety * 1000;
const fromIso2 = new Date(Math.max(0, fromMs)).toISOString();
try {
await hasuraRunSql(
cfg,
sqlUpsertCandlesFromTicks({ symbol, sourceKey, bucketSeconds: bs, fromIso: fromIso2, toIso: loopIso })
);
// best-effort: move last pointer close to now (actual max will lag by at most one bucket)
last.set(k, loopIso);
const prevPrune = lastPruneAt.get(k) || 0;
if (loopNow - prevPrune >= pruneEveryMs) {
const keepSeconds = Math.max(cfg.targetPoints * bs, (cfg.backfillDays > 0 ? cfg.backfillDays * 86400 : 0));
const cutoffIso = new Date(Math.max(0, loopNow - keepSeconds * 1000)).toISOString();
try {
await hasuraRunSql(cfg, sqlDeleteOlderCandles({ symbol, sourceKey, bucketSeconds: bs, cutoffIso }));
} finally {
lastPruneAt.set(k, loopNow);
}
}
} catch (err) {
console.error(
`[candles-cache-worker] update failed (${symbol}/${sourceKey || 'any'}/${bs}s): ${String(err?.message || err)}`
);
}
}
}
}
await sleep(cfg.pollMs);
}
}
main().catch((err) => {
console.error(String(err?.stack || err));
process.exitCode = 1;
});

View File

@@ -0,0 +1,50 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-depth-worker-drift
annotations:
argocd.argoproj.io/sync-wave: "6"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-depth-worker-drift
template:
metadata:
labels:
app.kubernetes.io/name: dlob-depth-worker-drift
spec:
containers:
- name: worker
image: node:20-slim
imagePullPolicy: IfNotPresent
env:
- name: HASURA_GRAPHQL_URL
value: http://hasura:8080/v1/graphql
- name: HASURA_ADMIN_SECRET
valueFrom:
secretKeyRef:
name: trade-hasura
key: HASURA_GRAPHQL_ADMIN_SECRET
- name: DLOB_SOURCE
value: drift
- name: DLOB_MARKETS
value: SOL-PERP,DOGE-PERP,JUP-PERP
- name: DLOB_POLL_MS
value: "1000"
- name: DLOB_DEPTH_BPS_BANDS
value: "5,10,20,50,100,200"
- name: PRICE_PRECISION
value: "1000000"
- name: BASE_PRECISION
value: "1000000000"
command: ["node", "/app/worker.mjs"]
volumeMounts:
- name: script
mountPath: /app/worker.mjs
subPath: worker.mjs
readOnly: true
volumes:
- name: script
configMap:
name: dlob-depth-worker-script

View File

@@ -0,0 +1,50 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-depth-worker
annotations:
argocd.argoproj.io/sync-wave: "6"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-depth-worker
template:
metadata:
labels:
app.kubernetes.io/name: dlob-depth-worker
spec:
containers:
- name: worker
image: node:20-slim
imagePullPolicy: IfNotPresent
env:
- name: HASURA_GRAPHQL_URL
value: http://hasura:8080/v1/graphql
- name: HASURA_ADMIN_SECRET
valueFrom:
secretKeyRef:
name: trade-hasura
key: HASURA_GRAPHQL_ADMIN_SECRET
- name: DLOB_SOURCE
value: mevnode
- name: DLOB_MARKETS
value: SOL-PERP,DOGE-PERP,JUP-PERP
- name: DLOB_POLL_MS
value: "1000"
- name: DLOB_DEPTH_BPS_BANDS
value: "5,10,20,50,100,200"
- name: PRICE_PRECISION
value: "1000000"
- name: BASE_PRECISION
value: "1000000000"
command: ["node", "/app/worker.mjs"]
volumeMounts:
- name: script
mountPath: /app/worker.mjs
subPath: worker.mjs
readOnly: true
volumes:
- name: script
configMap:
name: dlob-depth-worker-script

View File

@@ -0,0 +1,316 @@
import process from 'node:process';
import { setTimeout as sleep } from 'node:timers/promises';
function getIsoNow() {
return new Date().toISOString();
}
function clampInt(value, min, max, fallback) {
const n = Number.parseInt(String(value ?? ''), 10);
if (!Number.isInteger(n)) return fallback;
return Math.min(max, Math.max(min, n));
}
function envList(name, fallbackCsv) {
const raw = process.env[name] ?? fallbackCsv;
return String(raw)
.split(',')
.map((s) => s.trim())
.filter(Boolean);
}
function envIntList(name, fallbackCsv) {
const out = [];
for (const item of envList(name, fallbackCsv)) {
const n = Number.parseInt(item, 10);
if (!Number.isFinite(n)) continue;
out.push(n);
}
return out.length ? out : envList(name, fallbackCsv).map((v) => Number.parseInt(v, 10)).filter(Number.isFinite);
}
function toNumberOrNull(value) {
if (value == null) return null;
if (typeof value === 'number') return Number.isFinite(value) ? value : null;
if (typeof value === 'string') {
const s = value.trim();
if (!s) return null;
const n = Number(s);
return Number.isFinite(n) ? n : null;
}
return null;
}
function numStr(value) {
if (value == null) return null;
if (typeof value === 'number') return Number.isFinite(value) ? String(value) : null;
if (typeof value === 'string') return value.trim() || null;
return null;
}
function jsonNormalize(value) {
if (typeof value !== 'string') return value;
const s = value.trim();
if (!s) return null;
try {
return JSON.parse(s);
} catch {
return value;
}
}
function resolveConfig() {
const hasuraUrl = process.env.HASURA_GRAPHQL_URL || 'http://hasura:8080/v1/graphql';
const hasuraAdminSecret = process.env.HASURA_ADMIN_SECRET || process.env.HASURA_GRAPHQL_ADMIN_SECRET || undefined;
const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined;
const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode';
const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
const pollMs = clampInt(process.env.DLOB_POLL_MS, 250, 60_000, 1000);
const bandsBps = envIntList('DLOB_DEPTH_BPS_BANDS', '5,10,20,50,100,200');
const pricePrecision = Number(process.env.PRICE_PRECISION || 1_000_000);
const basePrecision = Number(process.env.BASE_PRECISION || 1_000_000_000);
if (!Number.isFinite(pricePrecision) || pricePrecision <= 0)
throw new Error(`Invalid PRICE_PRECISION: ${process.env.PRICE_PRECISION}`);
if (!Number.isFinite(basePrecision) || basePrecision <= 0)
throw new Error(`Invalid BASE_PRECISION: ${process.env.BASE_PRECISION}`);
return {
hasuraUrl,
hasuraAdminSecret,
hasuraAuthToken,
dlobSource,
markets,
pollMs,
bandsBps,
pricePrecision,
basePrecision,
};
}
async function graphqlRequest(cfg, query, variables) {
const headers = { 'content-type': 'application/json' };
if (cfg.hasuraAuthToken) {
headers.authorization = `Bearer ${cfg.hasuraAuthToken}`;
} else if (cfg.hasuraAdminSecret) {
headers['x-hasura-admin-secret'] = cfg.hasuraAdminSecret;
} else {
throw new Error('Missing Hasura auth (set HASURA_AUTH_TOKEN or HASURA_ADMIN_SECRET)');
}
const res = await fetch(cfg.hasuraUrl, {
method: 'POST',
headers,
body: JSON.stringify({ query, variables }),
signal: AbortSignal.timeout(10_000),
});
const text = await res.text();
if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`);
const json = JSON.parse(text);
if (json.errors?.length) throw new Error(json.errors.map((e) => e.message).join(' | '));
return json.data;
}
function parseLevels(raw, pricePrecision, basePrecision, side) {
const v = jsonNormalize(raw);
if (!Array.isArray(v)) return [];
const out = [];
for (const item of v) {
const priceInt = toNumberOrNull(item?.price);
const sizeInt = toNumberOrNull(item?.size);
if (priceInt == null || sizeInt == null) continue;
const price = priceInt / pricePrecision;
const size = sizeInt / basePrecision;
if (!Number.isFinite(price) || !Number.isFinite(size)) continue;
out.push({ price, size });
}
if (side === 'bid') out.sort((a, b) => b.price - a.price);
if (side === 'ask') out.sort((a, b) => a.price - b.price);
return out;
}
function computeMid(bestBid, bestAsk, markPrice, oraclePrice) {
if (bestBid != null && bestAsk != null) return (bestBid + bestAsk) / 2;
if (markPrice != null) return markPrice;
if (oraclePrice != null) return oraclePrice;
return null;
}
function computeBandDepth({ bids, asks, mid, bandBps }) {
if (mid == null || !(mid > 0)) {
return { bidBase: 0, askBase: 0, bidUsd: 0, askUsd: 0, imbalance: null };
}
const minBidPrice = mid * (1 - bandBps / 10_000);
const maxAskPrice = mid * (1 + bandBps / 10_000);
let bidBase = 0;
let askBase = 0;
let bidUsd = 0;
let askUsd = 0;
for (const lvl of bids) {
if (lvl.price < minBidPrice) break;
bidBase += lvl.size;
bidUsd += lvl.size * lvl.price;
}
for (const lvl of asks) {
if (lvl.price > maxAskPrice) break;
askBase += lvl.size;
askUsd += lvl.size * lvl.price;
}
const denom = bidUsd + askUsd;
const imbalance = denom > 0 ? (bidUsd - askUsd) / denom : null;
return { bidBase, askBase, bidUsd, askUsd, imbalance };
}
async function fetchL2Latest(cfg) {
const query = `
query DlobL2Latest($source: String!, $markets: [String!]!) {
dlob_l2_latest(where: {source: {_eq: $source}, market_name: {_in: $markets}}) {
source
market_name
market_type
market_index
ts
slot
mark_price
oracle_price
best_bid_price
best_ask_price
bids
asks
updated_at
}
}
`;
const data = await graphqlRequest(cfg, query, { source: cfg.dlobSource, markets: cfg.markets });
return Array.isArray(data?.dlob_l2_latest) ? data.dlob_l2_latest : [];
}
async function upsertDepth(cfg, rows) {
if (!rows.length) return;
const mutation = `
mutation UpsertDlobDepth($rows: [dlob_depth_bps_latest_insert_input!]!) {
insert_dlob_depth_bps_latest(
objects: $rows
on_conflict: {
constraint: dlob_depth_bps_latest_pkey
update_columns: [
market_type
market_index
ts
slot
mid_price
best_bid_price
best_ask_price
bid_base
ask_base
bid_usd
ask_usd
imbalance
raw
updated_at
]
}
) { affected_rows }
}
`;
await graphqlRequest(cfg, mutation, { rows });
}
async function main() {
const cfg = resolveConfig();
const lastUpdatedAtByMarket = new Map();
console.log(
JSON.stringify(
{
service: 'dlob-depth-worker',
startedAt: getIsoNow(),
hasuraUrl: cfg.hasuraUrl,
hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none',
dlobSource: cfg.dlobSource,
markets: cfg.markets,
pollMs: cfg.pollMs,
bandsBps: cfg.bandsBps,
pricePrecision: cfg.pricePrecision,
basePrecision: cfg.basePrecision,
},
null,
2
)
);
while (true) {
const rows = [];
try {
const l2Rows = await fetchL2Latest(cfg);
for (const l2 of l2Rows) {
const market = String(l2.market_name || '').trim();
if (!market) continue;
const updatedAt = l2.updated_at || null;
if (updatedAt && lastUpdatedAtByMarket.get(market) === updatedAt) continue;
if (updatedAt) lastUpdatedAtByMarket.set(market, updatedAt);
const bestBid = toNumberOrNull(l2.best_bid_price);
const bestAsk = toNumberOrNull(l2.best_ask_price);
const markPrice = toNumberOrNull(l2.mark_price);
const oraclePrice = toNumberOrNull(l2.oracle_price);
const mid = computeMid(bestBid, bestAsk, markPrice, oraclePrice);
const bids = parseLevels(l2.bids, cfg.pricePrecision, cfg.basePrecision, 'bid');
const asks = parseLevels(l2.asks, cfg.pricePrecision, cfg.basePrecision, 'ask');
for (const bandBps of cfg.bandsBps) {
const d = computeBandDepth({ bids, asks, mid, bandBps });
rows.push({
source: cfg.dlobSource,
market_name: market,
band_bps: bandBps,
market_type: l2.market_type ? String(l2.market_type) : 'perp',
market_index: typeof l2.market_index === 'number' ? l2.market_index : null,
ts: l2.ts == null ? null : String(l2.ts),
slot: l2.slot == null ? null : String(l2.slot),
mid_price: numStr(mid),
best_bid_price: numStr(bestBid),
best_ask_price: numStr(bestAsk),
bid_base: numStr(d.bidBase),
ask_base: numStr(d.askBase),
bid_usd: numStr(d.bidUsd),
ask_usd: numStr(d.askUsd),
imbalance: numStr(d.imbalance),
raw: {
ref: 'mid',
pricePrecision: cfg.pricePrecision,
basePrecision: cfg.basePrecision,
},
updated_at: updatedAt,
});
}
}
} catch (err) {
console.error(`[dlob-depth-worker] fetch/compute: ${String(err?.message || err)}`);
}
try {
await upsertDepth(cfg, rows);
} catch (err) {
console.error(`[dlob-depth-worker] upsert: ${String(err?.message || err)}`);
}
await sleep(cfg.pollMs);
}
}
main().catch((err) => {
console.error(String(err?.stack || err));
process.exitCode = 1;
});

View File

@@ -0,0 +1,50 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-slippage-worker-drift
annotations:
argocd.argoproj.io/sync-wave: "6"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-slippage-worker-drift
template:
metadata:
labels:
app.kubernetes.io/name: dlob-slippage-worker-drift
spec:
containers:
- name: worker
image: node:20-slim
imagePullPolicy: IfNotPresent
env:
- name: HASURA_GRAPHQL_URL
value: http://hasura:8080/v1/graphql
- name: HASURA_ADMIN_SECRET
valueFrom:
secretKeyRef:
name: trade-hasura
key: HASURA_GRAPHQL_ADMIN_SECRET
- name: DLOB_SOURCE
value: drift
- name: DLOB_MARKETS
value: SOL-PERP,DOGE-PERP,JUP-PERP
- name: DLOB_POLL_MS
value: "1000"
- name: DLOB_SLIPPAGE_SIZES_USD
value: "0.1,0.2,0.5,1,2,5,10,25,50,100,250,500,1000,5000,10000,50000"
- name: PRICE_PRECISION
value: "1000000"
- name: BASE_PRECISION
value: "1000000000"
command: ["node", "/app/worker.mjs"]
volumeMounts:
- name: script
mountPath: /app/worker.mjs
subPath: worker.mjs
readOnly: true
volumes:
- name: script
configMap:
name: dlob-slippage-worker-script

View File

@@ -0,0 +1,50 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-slippage-worker
annotations:
argocd.argoproj.io/sync-wave: "6"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-slippage-worker
template:
metadata:
labels:
app.kubernetes.io/name: dlob-slippage-worker
spec:
containers:
- name: worker
image: node:20-slim
imagePullPolicy: IfNotPresent
env:
- name: HASURA_GRAPHQL_URL
value: http://hasura:8080/v1/graphql
- name: HASURA_ADMIN_SECRET
valueFrom:
secretKeyRef:
name: trade-hasura
key: HASURA_GRAPHQL_ADMIN_SECRET
- name: DLOB_SOURCE
value: mevnode
- name: DLOB_MARKETS
value: SOL-PERP,DOGE-PERP,JUP-PERP
- name: DLOB_POLL_MS
value: "1000"
- name: DLOB_SLIPPAGE_SIZES_USD
value: "0.1,0.2,0.5,1,2,5,10,25,50,100,250,500,1000,5000,10000,50000"
- name: PRICE_PRECISION
value: "1000000"
- name: BASE_PRECISION
value: "1000000000"
command: ["node", "/app/worker.mjs"]
volumeMounts:
- name: script
mountPath: /app/worker.mjs
subPath: worker.mjs
readOnly: true
volumes:
- name: script
configMap:
name: dlob-slippage-worker-script

View File

@@ -0,0 +1,404 @@
import fs from 'node:fs';
import process from 'node:process';
import { setTimeout as sleep } from 'node:timers/promises';
function readJsonFile(filePath) {
try {
const raw = fs.readFileSync(filePath, 'utf8');
return JSON.parse(raw);
} catch {
return undefined;
}
}
function getIsoNow() {
return new Date().toISOString();
}
function clampInt(value, min, max, fallback) {
const n = Number.parseInt(String(value ?? ''), 10);
if (!Number.isInteger(n)) return fallback;
return Math.min(max, Math.max(min, n));
}
function envList(name, fallbackCsv) {
const raw = process.env[name] ?? fallbackCsv;
return String(raw)
.split(',')
.map((s) => s.trim())
.filter(Boolean);
}
function parsePositiveNumber(value) {
const n = Number.parseFloat(String(value ?? '').trim());
if (!Number.isFinite(n) || !(n > 0)) return null;
return n;
}
function resolveConfig() {
const tokensPath =
process.env.HASURA_TOKENS_FILE ||
process.env.TOKENS_FILE ||
process.env.HASURA_CONFIG_FILE ||
'/app/tokens/hasura.json';
const tokens = readJsonFile(tokensPath) || {};
const hasuraUrl =
process.env.HASURA_GRAPHQL_URL ||
tokens.graphqlUrl ||
tokens.apiUrl ||
'http://hasura:8080/v1/graphql';
const hasuraAdminSecret =
process.env.HASURA_ADMIN_SECRET ||
process.env.HASURA_GRAPHQL_ADMIN_SECRET ||
tokens.adminSecret ||
tokens.hasuraAdminSecret;
const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined;
const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode';
const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
const pollMs = clampInt(process.env.DLOB_POLL_MS, 250, 60_000, 1000);
const sizesUsd = envList('DLOB_SLIPPAGE_SIZES_USD', '10,25,50,100,250,500,1000')
.map(parsePositiveNumber)
.filter((n) => n != null)
.map((n) => n)
.filter((n, idx, arr) => arr.findIndex((x) => x === n) === idx)
.sort((a, b) => a - b);
const sizesUsdInt = sizesUsd.filter((n) => Number.isInteger(n));
const depthLevels = clampInt(process.env.DLOB_DEPTH, 1, 50, 25);
const pricePrecision = Number(process.env.PRICE_PRECISION || 1_000_000);
const basePrecision = Number(process.env.BASE_PRECISION || 1_000_000_000);
if (!Number.isFinite(pricePrecision) || pricePrecision <= 0) throw new Error(`Invalid PRICE_PRECISION: ${process.env.PRICE_PRECISION}`);
if (!Number.isFinite(basePrecision) || basePrecision <= 0) throw new Error(`Invalid BASE_PRECISION: ${process.env.BASE_PRECISION}`);
return {
hasuraUrl,
hasuraAdminSecret,
hasuraAuthToken,
dlobSource,
markets,
pollMs,
sizesUsd,
sizesUsdInt,
depthLevels,
pricePrecision,
basePrecision,
};
}
async function graphqlRequest(cfg, query, variables) {
const headers = { 'content-type': 'application/json' };
if (cfg.hasuraAuthToken) {
headers.authorization = `Bearer ${cfg.hasuraAuthToken}`;
} else if (cfg.hasuraAdminSecret) {
headers['x-hasura-admin-secret'] = cfg.hasuraAdminSecret;
} else {
throw new Error('Missing Hasura auth (set HASURA_AUTH_TOKEN or HASURA_ADMIN_SECRET or mount tokens/hasura.json)');
}
const res = await fetch(cfg.hasuraUrl, {
method: 'POST',
headers,
body: JSON.stringify({ query, variables }),
signal: AbortSignal.timeout(15_000),
});
const text = await res.text();
if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`);
const json = JSON.parse(text);
if (json.errors?.length) {
throw new Error(json.errors.map((e) => e.message).join(' | '));
}
return json.data;
}
function toNumberOrNull(value) {
if (value == null) return null;
if (typeof value === 'number') return Number.isFinite(value) ? value : null;
if (typeof value === 'string') {
const s = value.trim();
if (!s) return null;
const n = Number(s);
return Number.isFinite(n) ? n : null;
}
return null;
}
function normalizeLevels(raw) {
if (raw == null) return [];
if (Array.isArray(raw)) return raw;
if (typeof raw === 'string') {
const s = raw.trim();
if (!s) return [];
try {
const v = JSON.parse(s);
return Array.isArray(v) ? v : [];
} catch {
return [];
}
}
return [];
}
function parseScaledLevels(raw, pricePrecision, basePrecision) {
const levels = normalizeLevels(raw);
const out = [];
for (const it of levels) {
const priceInt = toNumberOrNull(it?.price);
const sizeInt = toNumberOrNull(it?.size);
if (priceInt == null || sizeInt == null) continue;
const price = priceInt / pricePrecision;
const base = sizeInt / basePrecision;
if (!Number.isFinite(price) || !Number.isFinite(base)) continue;
out.push({ price, base });
}
return out;
}
function simulateFill(levels, sizeUsd) {
let remainingUsd = sizeUsd;
let filledUsd = 0;
let filledBase = 0;
let totalQuoteUsd = 0;
let worstPrice = null;
let levelsConsumed = 0;
for (const l of levels) {
if (remainingUsd <= 0) break;
const levelUsd = l.base * l.price;
if (levelUsd <= 0) continue;
levelsConsumed += 1;
worstPrice = l.price;
const takeUsd = Math.min(remainingUsd, levelUsd);
const takeBase = takeUsd / l.price;
remainingUsd -= takeUsd;
filledUsd += takeUsd;
filledBase += takeBase;
totalQuoteUsd += takeUsd;
}
const vwapPrice = filledBase > 0 ? totalQuoteUsd / filledBase : null;
const fillPct = sizeUsd > 0 ? (filledUsd / sizeUsd) * 100 : null;
return {
filledUsd,
filledBase,
vwapPrice,
worstPrice,
levelsConsumed,
fillPct,
};
}
function impactBps({ side, mid, vwap }) {
if (mid == null || vwap == null || mid <= 0) return null;
if (side === 'buy') return ((vwap / mid) - 1) * 10_000;
if (side === 'sell') return (1 - (vwap / mid)) * 10_000;
return null;
}
async function main() {
const cfg = resolveConfig();
console.log(
JSON.stringify(
{
service: 'dlob-slippage-worker',
startedAt: getIsoNow(),
hasuraUrl: cfg.hasuraUrl,
hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none',
dlobSource: cfg.dlobSource,
markets: cfg.markets,
pollMs: cfg.pollMs,
sizesUsd: cfg.sizesUsd,
depthLevels: cfg.depthLevels,
},
null,
2
)
);
const lastSeenUpdatedAt = new Map(); // market -> updated_at
while (true) {
const updatedAt = getIsoNow();
try {
const query = `
query DlobL2Latest($source: String!, $markets: [String!]!) {
dlob_l2_latest(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
source
market_name
market_type
market_index
ts
slot
best_bid_price
best_ask_price
bids
asks
updated_at
}
}
`;
const data = await graphqlRequest(cfg, query, { source: cfg.dlobSource, markets: cfg.markets });
const rows = Array.isArray(data?.dlob_l2_latest) ? data.dlob_l2_latest : [];
const objectsV1 = [];
const objectsV2 = [];
for (const row of rows) {
const market = String(row?.market_name || '').trim();
if (!market) continue;
const rowUpdatedAt = row?.updated_at ?? null;
if (rowUpdatedAt && lastSeenUpdatedAt.get(market) === rowUpdatedAt) continue;
if (rowUpdatedAt) lastSeenUpdatedAt.set(market, rowUpdatedAt);
const bestBid = toNumberOrNull(row?.best_bid_price);
const bestAsk = toNumberOrNull(row?.best_ask_price);
if (bestBid == null || bestAsk == null) continue;
const mid = (bestBid + bestAsk) / 2;
if (!Number.isFinite(mid) || mid <= 0) continue;
const bids = parseScaledLevels(row?.bids, cfg.pricePrecision, cfg.basePrecision)
.slice()
.sort((a, b) => b.price - a.price)
.slice(0, cfg.depthLevels);
const asks = parseScaledLevels(row?.asks, cfg.pricePrecision, cfg.basePrecision)
.slice()
.sort((a, b) => a.price - b.price)
.slice(0, cfg.depthLevels);
for (const sizeUsd of cfg.sizesUsd) {
// buy consumes asks (worse prices as you go up)
{
const sim = simulateFill(asks, sizeUsd);
const baseObj = {
source: cfg.dlobSource,
market_name: market,
side: 'buy',
market_type: row?.market_type ?? 'perp',
market_index: row?.market_index ?? null,
ts: row?.ts == null ? null : String(row.ts),
slot: row?.slot == null ? null : String(row.slot),
mid_price: String(mid),
vwap_price: sim.vwapPrice == null ? null : String(sim.vwapPrice),
worst_price: sim.worstPrice == null ? null : String(sim.worstPrice),
filled_usd: String(sim.filledUsd),
filled_base: String(sim.filledBase),
impact_bps: impactBps({ side: 'buy', mid, vwap: sim.vwapPrice }),
levels_consumed: sim.levelsConsumed,
fill_pct: sim.fillPct == null ? null : String(sim.fillPct),
raw: { depthLevels: cfg.depthLevels },
updated_at: updatedAt,
};
objectsV2.push({ ...baseObj, size_usd: String(sizeUsd) });
if (Number.isInteger(sizeUsd)) objectsV1.push({ ...baseObj, size_usd: Math.trunc(sizeUsd) });
}
// sell consumes bids (worse prices as you go down)
{
const sim = simulateFill(bids, sizeUsd);
const baseObj = {
source: cfg.dlobSource,
market_name: market,
side: 'sell',
market_type: row?.market_type ?? 'perp',
market_index: row?.market_index ?? null,
ts: row?.ts == null ? null : String(row.ts),
slot: row?.slot == null ? null : String(row.slot),
mid_price: String(mid),
vwap_price: sim.vwapPrice == null ? null : String(sim.vwapPrice),
worst_price: sim.worstPrice == null ? null : String(sim.worstPrice),
filled_usd: String(sim.filledUsd),
filled_base: String(sim.filledBase),
impact_bps: impactBps({ side: 'sell', mid, vwap: sim.vwapPrice }),
levels_consumed: sim.levelsConsumed,
fill_pct: sim.fillPct == null ? null : String(sim.fillPct),
raw: { depthLevels: cfg.depthLevels },
updated_at: updatedAt,
};
objectsV2.push({ ...baseObj, size_usd: String(sizeUsd) });
if (Number.isInteger(sizeUsd)) objectsV1.push({ ...baseObj, size_usd: Math.trunc(sizeUsd) });
}
}
}
if (objectsV1.length) {
const mutation = `
mutation UpsertSlippageV1($rows: [dlob_slippage_latest_insert_input!]!) {
insert_dlob_slippage_latest(
objects: $rows
on_conflict: {
constraint: dlob_slippage_latest_pkey
update_columns: [
market_type
market_index
ts
slot
mid_price
vwap_price
worst_price
filled_usd
filled_base
impact_bps
levels_consumed
fill_pct
raw
updated_at
]
}
) { affected_rows }
}
`;
await graphqlRequest(cfg, mutation, { rows: objectsV1 });
}
if (objectsV2.length) {
const mutation = `
mutation UpsertSlippageV2($rows: [dlob_slippage_latest_v2_insert_input!]!) {
insert_dlob_slippage_latest_v2(
objects: $rows
on_conflict: {
constraint: dlob_slippage_latest_v2_pkey
update_columns: [
market_type
market_index
ts
slot
mid_price
vwap_price
worst_price
filled_usd
filled_base
impact_bps
levels_consumed
fill_pct
raw
updated_at
]
}
) { affected_rows }
}
`;
await graphqlRequest(cfg, mutation, { rows: objectsV2 });
}
} catch (err) {
console.error(`[dlob-slippage-worker] ${String(err?.message || err)}`);
}
await sleep(cfg.pollMs);
}
}
main().catch((err) => {
console.error(String(err?.stack || err));
process.exitCode = 1;
});

View File

@@ -0,0 +1,44 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-ts-archiver-drift
annotations:
argocd.argoproj.io/sync-wave: "6"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-ts-archiver-drift
template:
metadata:
labels:
app.kubernetes.io/name: dlob-ts-archiver-drift
spec:
containers:
- name: worker
image: node:20-slim
imagePullPolicy: IfNotPresent
env:
- name: HASURA_GRAPHQL_URL
value: http://hasura:8080/v1/graphql
- name: HASURA_ADMIN_SECRET
valueFrom:
secretKeyRef:
name: trade-hasura
key: HASURA_GRAPHQL_ADMIN_SECRET
- name: DLOB_SOURCE
value: drift
- name: DLOB_MARKETS
value: SOL-PERP,DOGE-PERP,JUP-PERP
- name: DLOB_TS_POLL_MS
value: "1000"
command: ["node", "/app/worker.mjs"]
volumeMounts:
- name: script
mountPath: /app/worker.mjs
subPath: worker.mjs
readOnly: true
volumes:
- name: script
configMap:
name: dlob-ts-archiver-script

View File

@@ -0,0 +1,44 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-ts-archiver
annotations:
argocd.argoproj.io/sync-wave: "6"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-ts-archiver
template:
metadata:
labels:
app.kubernetes.io/name: dlob-ts-archiver
spec:
containers:
- name: worker
image: node:20-slim
imagePullPolicy: IfNotPresent
env:
- name: HASURA_GRAPHQL_URL
value: http://hasura:8080/v1/graphql
- name: HASURA_ADMIN_SECRET
valueFrom:
secretKeyRef:
name: trade-hasura
key: HASURA_GRAPHQL_ADMIN_SECRET
- name: DLOB_SOURCE
value: mevnode
- name: DLOB_MARKETS
value: SOL-PERP,DOGE-PERP,JUP-PERP
- name: DLOB_TS_POLL_MS
value: "1000"
command: ["node", "/app/worker.mjs"]
volumeMounts:
- name: script
mountPath: /app/worker.mjs
subPath: worker.mjs
readOnly: true
volumes:
- name: script
configMap:
name: dlob-ts-archiver-script

View File

@@ -0,0 +1,254 @@
import fs from 'node:fs';
import process from 'node:process';
import { setTimeout as sleep } from 'node:timers/promises';
function readJsonFile(filePath) {
try {
const raw = fs.readFileSync(filePath, 'utf8');
return JSON.parse(raw);
} catch {
return undefined;
}
}
function getIsoNow() {
return new Date().toISOString();
}
function clampInt(value, min, max, fallback) {
const n = Number.parseInt(String(value ?? ''), 10);
if (!Number.isInteger(n)) return fallback;
return Math.min(max, Math.max(min, n));
}
function envList(name, fallbackCsv) {
const raw = process.env[name] ?? fallbackCsv;
return String(raw)
.split(',')
.map((s) => s.trim())
.filter(Boolean);
}
function resolveConfig() {
const tokensPath =
process.env.HASURA_TOKENS_FILE ||
process.env.TOKENS_FILE ||
process.env.HASURA_CONFIG_FILE ||
'/app/tokens/hasura.json';
const tokens = readJsonFile(tokensPath) || {};
const hasuraUrl =
process.env.HASURA_GRAPHQL_URL ||
tokens.graphqlUrl ||
tokens.apiUrl ||
'http://hasura:8080/v1/graphql';
const hasuraAdminSecret =
process.env.HASURA_ADMIN_SECRET ||
process.env.HASURA_GRAPHQL_ADMIN_SECRET ||
tokens.adminSecret ||
tokens.hasuraAdminSecret;
const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined;
const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode';
const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
const pollMs = clampInt(process.env.DLOB_TS_POLL_MS, 500, 60_000, 1000);
return { hasuraUrl, hasuraAdminSecret, hasuraAuthToken, dlobSource, markets, pollMs };
}
async function graphqlRequest(cfg, query, variables) {
const headers = { 'content-type': 'application/json' };
if (cfg.hasuraAuthToken) {
headers.authorization = `Bearer ${cfg.hasuraAuthToken}`;
} else if (cfg.hasuraAdminSecret) {
headers['x-hasura-admin-secret'] = cfg.hasuraAdminSecret;
} else {
throw new Error('Missing Hasura auth (set HASURA_AUTH_TOKEN or HASURA_ADMIN_SECRET or mount tokens/hasura.json)');
}
const res = await fetch(cfg.hasuraUrl, {
method: 'POST',
headers,
body: JSON.stringify({ query, variables }),
signal: AbortSignal.timeout(15_000),
});
const text = await res.text();
if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`);
const json = JSON.parse(text);
if (json.errors?.length) {
throw new Error(json.errors.map((e) => e.message).join(' | '));
}
return json.data;
}
function mapBigint(v) {
if (v == null) return null;
if (typeof v === 'number') return Number.isFinite(v) ? String(Math.trunc(v)) : null;
if (typeof v === 'string') return v.trim() || null;
return null;
}
async function main() {
const cfg = resolveConfig();
console.log(
JSON.stringify(
{
service: 'dlob-ts-archiver',
startedAt: getIsoNow(),
hasuraUrl: cfg.hasuraUrl,
hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none',
dlobSource: cfg.dlobSource,
markets: cfg.markets,
pollMs: cfg.pollMs,
},
null,
2
)
);
while (true) {
const now = getIsoNow();
try {
const query = `
query Latest($source: String!, $markets: [String!]!) {
dlob_stats_latest(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
market_name market_type market_index ts slot
mark_price oracle_price best_bid_price best_ask_price mid_price
spread_abs spread_bps depth_levels depth_bid_base depth_ask_base depth_bid_usd depth_ask_usd imbalance
raw
}
dlob_depth_bps_latest(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
market_name band_bps market_type market_index ts slot
mid_price best_bid_price best_ask_price bid_base ask_base bid_usd ask_usd imbalance
raw
}
dlob_slippage_latest(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
market_name side size_usd market_type market_index ts slot
mid_price vwap_price worst_price filled_usd filled_base impact_bps levels_consumed fill_pct
raw
}
dlob_slippage_latest_v2(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
market_name side size_usd market_type market_index ts slot
mid_price vwap_price worst_price filled_usd filled_base impact_bps levels_consumed fill_pct
raw
}
}
`;
const data = await graphqlRequest(cfg, query, { source: cfg.dlobSource, markets: cfg.markets });
const statsRows = (data?.dlob_stats_latest || []).map((r) => ({
ts: now,
source: cfg.dlobSource,
market_name: r.market_name,
market_type: r.market_type,
market_index: r.market_index ?? null,
source_ts: mapBigint(r.ts),
slot: mapBigint(r.slot),
mark_price: r.mark_price ?? null,
oracle_price: r.oracle_price ?? null,
best_bid_price: r.best_bid_price ?? null,
best_ask_price: r.best_ask_price ?? null,
mid_price: r.mid_price ?? null,
spread_abs: r.spread_abs ?? null,
spread_bps: r.spread_bps ?? null,
depth_levels: r.depth_levels ?? null,
depth_bid_base: r.depth_bid_base ?? null,
depth_ask_base: r.depth_ask_base ?? null,
depth_bid_usd: r.depth_bid_usd ?? null,
depth_ask_usd: r.depth_ask_usd ?? null,
imbalance: r.imbalance ?? null,
raw: r.raw ?? null,
}));
const depthRows = (data?.dlob_depth_bps_latest || []).map((r) => ({
ts: now,
source: cfg.dlobSource,
market_name: r.market_name,
band_bps: r.band_bps,
market_type: r.market_type,
market_index: r.market_index ?? null,
source_ts: mapBigint(r.ts),
slot: mapBigint(r.slot),
mid_price: r.mid_price ?? null,
best_bid_price: r.best_bid_price ?? null,
best_ask_price: r.best_ask_price ?? null,
bid_base: r.bid_base ?? null,
ask_base: r.ask_base ?? null,
bid_usd: r.bid_usd ?? null,
ask_usd: r.ask_usd ?? null,
imbalance: r.imbalance ?? null,
raw: r.raw ?? null,
}));
const slippageRows = (data?.dlob_slippage_latest || []).map((r) => ({
ts: now,
source: cfg.dlobSource,
market_name: r.market_name,
side: r.side,
size_usd: r.size_usd,
market_type: r.market_type,
market_index: r.market_index ?? null,
source_ts: mapBigint(r.ts),
slot: mapBigint(r.slot),
mid_price: r.mid_price ?? null,
vwap_price: r.vwap_price ?? null,
worst_price: r.worst_price ?? null,
filled_usd: r.filled_usd ?? null,
filled_base: r.filled_base ?? null,
impact_bps: r.impact_bps ?? null,
levels_consumed: r.levels_consumed ?? null,
fill_pct: r.fill_pct ?? null,
raw: r.raw ?? null,
}));
const slippageRowsV2 = (data?.dlob_slippage_latest_v2 || []).map((r) => ({
ts: now,
source: cfg.dlobSource,
market_name: r.market_name,
side: r.side,
size_usd: r.size_usd,
market_type: r.market_type,
market_index: r.market_index ?? null,
source_ts: mapBigint(r.ts),
slot: mapBigint(r.slot),
mid_price: r.mid_price ?? null,
vwap_price: r.vwap_price ?? null,
worst_price: r.worst_price ?? null,
filled_usd: r.filled_usd ?? null,
filled_base: r.filled_base ?? null,
impact_bps: r.impact_bps ?? null,
levels_consumed: r.levels_consumed ?? null,
fill_pct: r.fill_pct ?? null,
raw: r.raw ?? null,
}));
const mutation = `
mutation InsertTs(
$stats: [dlob_stats_ts_insert_input!]!
$depth: [dlob_depth_bps_ts_insert_input!]!
$slip: [dlob_slippage_ts_insert_input!]!
$slipV2: [dlob_slippage_ts_v2_insert_input!]!
) {
insert_dlob_stats_ts(objects: $stats) { affected_rows }
insert_dlob_depth_bps_ts(objects: $depth) { affected_rows }
insert_dlob_slippage_ts(objects: $slip) { affected_rows }
insert_dlob_slippage_ts_v2(objects: $slipV2) { affected_rows }
}
`;
await graphqlRequest(cfg, mutation, { stats: statsRows, depth: depthRows, slip: slippageRows, slipV2: slippageRowsV2 });
} catch (err) {
console.error(`[dlob-ts-archiver] ${String(err?.message || err)}`);
}
await sleep(cfg.pollMs);
}
}
main().catch((err) => {
console.error(String(err?.stack || err));
process.exitCode = 1;
});

View File

@@ -0,0 +1,52 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-worker-drift
annotations:
argocd.argoproj.io/sync-wave: "5"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-worker-drift
template:
metadata:
labels:
app.kubernetes.io/name: dlob-worker-drift
spec:
hostNetwork: true
dnsPolicy: ClusterFirstWithHostNet
containers:
- name: worker
image: node:20-slim
imagePullPolicy: IfNotPresent
env:
- name: HASURA_GRAPHQL_URL
value: http://hasura:8080/v1/graphql
- name: HASURA_ADMIN_SECRET
valueFrom:
secretKeyRef:
name: trade-hasura
key: HASURA_GRAPHQL_ADMIN_SECRET
- name: DLOB_SOURCE
value: drift
- name: DLOB_HTTP_URL
value: https://dlob.drift.trade
- name: DLOB_FORCE_IPV6
value: "true"
- name: DLOB_MARKETS
value: SOL-PERP,DOGE-PERP,JUP-PERP
- name: DLOB_POLL_MS
value: "500"
- name: DLOB_DEPTH
value: "10"
command: ["node", "/app/worker.mjs"]
volumeMounts:
- name: script
mountPath: /app/worker.mjs
subPath: worker.mjs
readOnly: true
volumes:
- name: script
configMap:
name: dlob-worker-script

View File

@@ -0,0 +1,48 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-worker
annotations:
argocd.argoproj.io/sync-wave: "5"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-worker
template:
metadata:
labels:
app.kubernetes.io/name: dlob-worker
spec:
containers:
- name: worker
image: node:20-slim
imagePullPolicy: IfNotPresent
env:
- name: HASURA_GRAPHQL_URL
value: http://hasura:8080/v1/graphql
- name: HASURA_ADMIN_SECRET
valueFrom:
secretKeyRef:
name: trade-hasura
key: HASURA_GRAPHQL_ADMIN_SECRET
- name: DLOB_SOURCE
value: mevnode
- name: DLOB_HTTP_URL
value: http://dlob-server:6969
- name: DLOB_MARKETS
value: SOL-PERP,DOGE-PERP,JUP-PERP
- name: DLOB_POLL_MS
value: "500"
- name: DLOB_DEPTH
value: "10"
command: ["node", "/app/worker.mjs"]
volumeMounts:
- name: script
mountPath: /app/worker.mjs
subPath: worker.mjs
readOnly: true
volumes:
- name: script
configMap:
name: dlob-worker-script

View File

@@ -0,0 +1,435 @@
import fs from 'node:fs';
import * as http from 'node:http';
import * as https from 'node:https';
import process from 'node:process';
import { setTimeout as sleep } from 'node:timers/promises';
function readJsonFile(filePath) {
try {
const raw = fs.readFileSync(filePath, 'utf8');
return JSON.parse(raw);
} catch {
return undefined;
}
}
function getIsoNow() {
return new Date().toISOString();
}
function clampInt(value, min, max, fallback) {
const n = Number.parseInt(String(value ?? ''), 10);
if (!Number.isInteger(n)) return fallback;
return Math.min(max, Math.max(min, n));
}
function envList(name, fallbackCsv) {
const raw = process.env[name] ?? fallbackCsv;
return String(raw)
.split(',')
.map((s) => s.trim())
.filter(Boolean);
}
function envBool(name, fallback = false) {
const raw = process.env[name];
if (raw == null) return fallback;
const v = String(raw).trim().toLowerCase();
if (['1', 'true', 'yes', 'y', 'on'].includes(v)) return true;
if (['0', 'false', 'no', 'n', 'off'].includes(v)) return false;
return fallback;
}
function resolveConfig() {
const tokensPath =
process.env.HASURA_TOKENS_FILE ||
process.env.TOKENS_FILE ||
process.env.HASURA_CONFIG_FILE ||
'/app/tokens/hasura.json';
const tokens = readJsonFile(tokensPath) || {};
const hasuraUrl =
process.env.HASURA_GRAPHQL_URL ||
tokens.graphqlUrl ||
tokens.apiUrl ||
'http://hasura:8080/v1/graphql';
const hasuraAdminSecret =
process.env.HASURA_ADMIN_SECRET ||
process.env.HASURA_GRAPHQL_ADMIN_SECRET ||
tokens.adminSecret ||
tokens.hasuraAdminSecret;
const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined;
const dlobHttpBase = String(process.env.DLOB_HTTP_URL || process.env.DLOB_HTTP_BASE || 'https://dlob.drift.trade')
.trim()
.replace(/\/$/, '');
const dlobForceIpv6 = envBool('DLOB_FORCE_IPV6', false);
const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode';
const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
const depth = clampInt(process.env.DLOB_DEPTH, 1, 50, 10);
const pollMs = clampInt(process.env.DLOB_POLL_MS, 100, 10_000, 500);
const pricePrecision = Number(process.env.PRICE_PRECISION || 1_000_000);
const basePrecision = Number(process.env.BASE_PRECISION || 1_000_000_000);
if (!Number.isFinite(pricePrecision) || pricePrecision <= 0)
throw new Error(`Invalid PRICE_PRECISION: ${process.env.PRICE_PRECISION}`);
if (!Number.isFinite(basePrecision) || basePrecision <= 0)
throw new Error(`Invalid BASE_PRECISION: ${process.env.BASE_PRECISION}`);
return {
hasuraUrl,
hasuraAdminSecret,
hasuraAuthToken,
dlobSource,
dlobHttpBase,
dlobForceIpv6,
markets,
depth,
pollMs,
pricePrecision,
basePrecision,
};
}
async function requestText(url, { timeoutMs, family } = {}) {
const u = new URL(url);
const client = u.protocol === 'https:' ? https : http;
const port = u.port ? Number.parseInt(u.port, 10) : u.protocol === 'https:' ? 443 : 80;
if (!Number.isFinite(port)) throw new Error(`Invalid port for url: ${url}`);
return await new Promise((resolve, reject) => {
const req = client.request(
{
protocol: u.protocol,
hostname: u.hostname,
port,
path: `${u.pathname}${u.search}`,
method: 'GET',
family,
servername: u.hostname,
headers: {
accept: 'application/json',
},
},
(res) => {
let data = '';
res.setEncoding('utf8');
res.on('data', (chunk) => {
data += chunk;
});
res.on('end', () => {
resolve({ status: res.statusCode ?? 0, text: data });
});
}
);
req.on('error', reject);
req.setTimeout(timeoutMs ?? 5_000, () => {
req.destroy(new Error(`Timeout after ${timeoutMs ?? 5_000}ms`));
});
req.end();
});
}
async function graphqlRequest(cfg, query, variables) {
const headers = { 'content-type': 'application/json' };
if (cfg.hasuraAuthToken) {
headers.authorization = `Bearer ${cfg.hasuraAuthToken}`;
} else if (cfg.hasuraAdminSecret) {
headers['x-hasura-admin-secret'] = cfg.hasuraAdminSecret;
} else {
throw new Error('Missing Hasura auth (set HASURA_AUTH_TOKEN or HASURA_ADMIN_SECRET or mount tokens/hasura.json)');
}
const res = await fetch(cfg.hasuraUrl, {
method: 'POST',
headers,
body: JSON.stringify({ query, variables }),
signal: AbortSignal.timeout(10_000),
});
const text = await res.text();
if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`);
const json = JSON.parse(text);
if (json.errors?.length) {
throw new Error(json.errors.map((e) => e.message).join(' | '));
}
return json.data;
}
function toNumberOrNull(value) {
if (value == null) return null;
if (typeof value === 'number') return Number.isFinite(value) ? value : null;
if (typeof value === 'string') {
const s = value.trim();
if (!s) return null;
const n = Number(s);
return Number.isFinite(n) ? n : null;
}
return null;
}
function numStr(value) {
if (value == null) return null;
if (typeof value === 'number') return Number.isFinite(value) ? String(value) : null;
if (typeof value === 'string') return value.trim() || null;
return null;
}
function parseScaled(valueRaw, scale) {
const n = toNumberOrNull(valueRaw);
if (n == null) return null;
return n / scale;
}
function computeStats({ l2, depth, pricePrecision, basePrecision }) {
const bids = Array.isArray(l2?.bids) ? l2.bids : [];
const asks = Array.isArray(l2?.asks) ? l2.asks : [];
const bestBid = parseScaled(l2?.bestBidPrice ?? bids?.[0]?.price, pricePrecision);
const bestAsk = parseScaled(l2?.bestAskPrice ?? asks?.[0]?.price, pricePrecision);
const markPrice = parseScaled(l2?.markPrice, pricePrecision);
const oraclePrice = parseScaled(l2?.oracleData?.price ?? l2?.oracle, pricePrecision);
const mid = bestBid != null && bestAsk != null ? (bestBid + bestAsk) / 2 : null;
const spreadAbs = bestBid != null && bestAsk != null ? bestAsk - bestBid : null;
const spreadBps = spreadAbs != null && mid != null && mid > 0 ? (spreadAbs / mid) * 10_000 : null;
const levels = Math.max(1, depth);
let bidBase = 0;
let askBase = 0;
let bidUsd = 0;
let askUsd = 0;
for (let i = 0; i < Math.min(levels, bids.length); i += 1) {
const p = parseScaled(bids[i]?.price, pricePrecision);
const s = toNumberOrNull(bids[i]?.size);
if (p == null || s == null) continue;
const base = s / basePrecision;
bidBase += base;
bidUsd += base * p;
}
for (let i = 0; i < Math.min(levels, asks.length); i += 1) {
const p = parseScaled(asks[i]?.price, pricePrecision);
const s = toNumberOrNull(asks[i]?.size);
if (p == null || s == null) continue;
const base = s / basePrecision;
askBase += base;
askUsd += base * p;
}
const denom = bidUsd + askUsd;
const imbalance = denom > 0 ? (bidUsd - askUsd) / denom : null;
return {
bestBid,
bestAsk,
mid,
spreadAbs,
spreadBps,
markPrice,
oraclePrice,
depthLevels: levels,
bidBase,
askBase,
bidUsd,
askUsd,
imbalance,
};
}
function l2ToInsertObject({ dlobSource, l2, updatedAt, pricePrecision }) {
return {
source: dlobSource,
market_name: String(l2.marketName),
market_type: String(l2.marketType || 'perp'),
market_index: typeof l2.marketIndex === 'number' ? l2.marketIndex : null,
ts: l2.ts == null ? null : String(l2.ts),
slot: l2.slot == null ? null : String(l2.slot),
mark_price: numStr(parseScaled(l2.markPrice, pricePrecision)),
oracle_price: numStr(parseScaled(l2.oracleData?.price ?? l2.oracle, pricePrecision)),
best_bid_price: numStr(parseScaled(l2.bestBidPrice, pricePrecision)),
best_ask_price: numStr(parseScaled(l2.bestAskPrice, pricePrecision)),
bids: l2.bids ?? null,
asks: l2.asks ?? null,
raw: l2 ?? null,
updated_at: updatedAt,
};
}
function statsToInsertObject({ dlobSource, l2, stats, updatedAt }) {
return {
source: dlobSource,
market_name: String(l2.marketName),
market_type: String(l2.marketType || 'perp'),
market_index: typeof l2.marketIndex === 'number' ? l2.marketIndex : null,
ts: l2.ts == null ? null : String(l2.ts),
slot: l2.slot == null ? null : String(l2.slot),
mark_price: stats.markPrice == null ? null : String(stats.markPrice),
oracle_price: stats.oraclePrice == null ? null : String(stats.oraclePrice),
best_bid_price: stats.bestBid == null ? null : String(stats.bestBid),
best_ask_price: stats.bestAsk == null ? null : String(stats.bestAsk),
mid_price: stats.mid == null ? null : String(stats.mid),
spread_abs: stats.spreadAbs == null ? null : String(stats.spreadAbs),
spread_bps: stats.spreadBps == null ? null : String(stats.spreadBps),
depth_levels: stats.depthLevels,
depth_bid_base: Number.isFinite(stats.bidBase) ? String(stats.bidBase) : null,
depth_ask_base: Number.isFinite(stats.askBase) ? String(stats.askBase) : null,
depth_bid_usd: Number.isFinite(stats.bidUsd) ? String(stats.bidUsd) : null,
depth_ask_usd: Number.isFinite(stats.askUsd) ? String(stats.askUsd) : null,
imbalance: stats.imbalance == null ? null : String(stats.imbalance),
raw: {
spreadPct: l2.spreadPct ?? null,
spreadQuote: l2.spreadQuote ?? null,
},
updated_at: updatedAt,
};
}
async function fetchL2(cfg, marketName) {
const u = new URL(`${cfg.dlobHttpBase}/l2`);
u.searchParams.set('marketName', marketName);
u.searchParams.set('depth', String(cfg.depth));
const url = u.toString();
if (cfg.dlobForceIpv6) {
const { status, text } = await requestText(url, { timeoutMs: 5_000, family: 6 });
if (status < 200 || status >= 300) throw new Error(`DLOB HTTP ${status}: ${text}`);
return JSON.parse(text);
}
const res = await fetch(url, { signal: AbortSignal.timeout(5_000) });
const text = await res.text();
if (!res.ok) throw new Error(`DLOB HTTP ${res.status}: ${text}`);
return JSON.parse(text);
}
async function upsertBatch(cfg, l2Objects, statsObjects) {
if (!l2Objects.length && !statsObjects.length) return;
const mutation = `
mutation UpsertDlob($l2: [dlob_l2_latest_insert_input!]!, $stats: [dlob_stats_latest_insert_input!]!) {
insert_dlob_l2_latest(
objects: $l2
on_conflict: {
constraint: dlob_l2_latest_pkey
update_columns: [
market_type
market_index
ts
slot
mark_price
oracle_price
best_bid_price
best_ask_price
bids
asks
raw
updated_at
]
}
) { affected_rows }
insert_dlob_stats_latest(
objects: $stats
on_conflict: {
constraint: dlob_stats_latest_pkey
update_columns: [
market_type
market_index
ts
slot
mark_price
oracle_price
best_bid_price
best_ask_price
mid_price
spread_abs
spread_bps
depth_levels
depth_bid_base
depth_ask_base
depth_bid_usd
depth_ask_usd
imbalance
raw
updated_at
]
}
) { affected_rows }
}
`;
await graphqlRequest(cfg, mutation, { l2: l2Objects, stats: statsObjects });
}
async function main() {
const cfg = resolveConfig();
const lastTsByMarket = new Map();
console.log(
JSON.stringify(
{
service: 'dlob-worker',
startedAt: getIsoNow(),
hasuraUrl: cfg.hasuraUrl,
hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none',
dlobSource: cfg.dlobSource,
dlobHttpBase: cfg.dlobHttpBase,
dlobForceIpv6: cfg.dlobForceIpv6,
markets: cfg.markets,
depth: cfg.depth,
pollMs: cfg.pollMs,
},
null,
2
)
);
while (true) {
const updatedAt = getIsoNow();
const results = await Promise.allSettled(cfg.markets.map((m) => fetchL2(cfg, m)));
const l2Objects = [];
const statsObjects = [];
for (let i = 0; i < results.length; i += 1) {
const market = cfg.markets[i];
const r = results[i];
if (r.status !== 'fulfilled') {
console.error(`[dlob-worker] fetch ${market}: ${String(r.reason?.message || r.reason)}`);
continue;
}
const l2 = r.value;
if (!l2?.marketName) continue;
const ts = l2.ts == null ? null : String(l2.ts);
if (ts != null && lastTsByMarket.get(l2.marketName) === ts) continue;
if (ts != null) lastTsByMarket.set(l2.marketName, ts);
const stats = computeStats({
l2,
depth: cfg.depth,
pricePrecision: cfg.pricePrecision,
basePrecision: cfg.basePrecision,
});
l2Objects.push(l2ToInsertObject({ dlobSource: cfg.dlobSource, l2, updatedAt, pricePrecision: cfg.pricePrecision }));
statsObjects.push(statsToInsertObject({ dlobSource: cfg.dlobSource, l2, stats, updatedAt }));
}
try {
await upsertBatch(cfg, l2Objects, statsObjects);
} catch (err) {
console.error(`[dlob-worker] upsert: ${String(err?.message || err)}`);
}
await sleep(cfg.pollMs);
}
}
main().catch((err) => {
console.error(String(err?.stack || err));
process.exitCode = 1;
});

View File

@@ -0,0 +1,73 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-publisher
annotations:
argocd.argoproj.io/sync-wave: "4"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-publisher
template:
metadata:
labels:
app.kubernetes.io/name: dlob-publisher
spec:
imagePullSecrets:
- name: gitea-registry
containers:
- name: publisher
image: gitea.mpabi.pl/trade/trade-dlob-server:sha-8a378b7-lite-l2
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 8080
env:
- name: RUNNING_LOCAL
value: "true"
- name: LOCAL_CACHE
value: "true"
- name: ENV
value: mainnet-beta
- name: USE_WEBSOCKET
value: "true"
- name: USE_ORDER_SUBSCRIBER
value: "true"
- name: DISABLE_GPA_REFRESH
value: "true"
- name: ELASTICACHE_HOST
value: dlob-redis
- name: ELASTICACHE_PORT
value: "6379"
- name: REDIS_CLIENT
value: DLOB
- name: PERP_MARKETS_TO_LOAD
value: "0,7,24"
- name: ENDPOINT
valueFrom:
secretKeyRef:
name: trade-dlob-rpc
key: ENDPOINT
- name: WS_ENDPOINT
valueFrom:
secretKeyRef:
name: trade-dlob-rpc
key: WS_ENDPOINT
command: ["node", "/lib/publishers/dlobPublisher.js"]
readinessProbe:
httpGet:
path: /startup
port: http
initialDelaySeconds: 120
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 30
livenessProbe:
httpGet:
path: /health
port: http
initialDelaySeconds: 240
periodSeconds: 20
timeoutSeconds: 3
failureThreshold: 10

View File

@@ -0,0 +1,79 @@
apiVersion: v1
kind: Service
metadata:
name: dlob-redis
annotations:
argocd.argoproj.io/sync-wave: "3"
spec:
selector:
app.kubernetes.io/name: dlob-redis
ports:
- name: redis
port: 6379
targetPort: redis
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-redis
annotations:
argocd.argoproj.io/sync-wave: "3"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-redis
template:
metadata:
labels:
app.kubernetes.io/name: dlob-redis
spec:
containers:
- name: redis
image: redis:7-alpine
imagePullPolicy: IfNotPresent
env:
- name: POD_IP
valueFrom:
fieldRef:
fieldPath: status.podIP
ports:
- name: redis
containerPort: 6379
# DLOB redis client uses ioredis Cluster when RUNNING_LOCAL=true and LOCAL_CACHE=true.
# We run a single-node Redis Cluster (no TLS) and assign all slots on startup.
command: ["/bin/sh", "-lc"]
args:
- |
exec redis-server \
--save "" \
--appendonly no \
--protected-mode no \
--bind 0.0.0.0 \
--cluster-enabled yes \
--cluster-config-file /data/nodes.conf \
--cluster-node-timeout 5000 \
--cluster-require-full-coverage no \
--cluster-announce-ip "${POD_IP}" \
--cluster-announce-port 6379 \
--cluster-announce-bus-port 16379
lifecycle:
postStart:
exec:
command:
- /bin/sh
- -lc
- |
set -e
for i in $(seq 1 60); do
redis-cli -h 127.0.0.1 -p 6379 ping >/dev/null 2>&1 && break
sleep 1
done
# If cluster is already initialized, do nothing.
if redis-cli -h 127.0.0.1 -p 6379 cluster info 2>/dev/null | grep -q 'cluster_slots_assigned:16384'; then
exit 0
fi
# Redis 7+ supports CLUSTER ADDSLOTSRANGE.
redis-cli -h 127.0.0.1 -p 6379 cluster addslotsrange 0 16383 || true

View File

@@ -0,0 +1,63 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-server
annotations:
argocd.argoproj.io/sync-wave: "4"
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-server
template:
metadata:
labels:
app.kubernetes.io/name: dlob-server
spec:
imagePullSecrets:
- name: gitea-registry
containers:
- name: server
image: gitea.mpabi.pl/trade/trade-dlob-server:sha-8a378b7-lite-l2
imagePullPolicy: IfNotPresent
ports:
- name: http
containerPort: 6969
env:
- name: RUNNING_LOCAL
value: "true"
- name: LOCAL_CACHE
value: "true"
- name: ENV
value: mainnet-beta
- name: PORT
value: "6969"
- name: ELASTICACHE_HOST
value: dlob-redis
- name: ELASTICACHE_PORT
value: "6379"
- name: REDIS_CLIENT
value: DLOB
- name: ENDPOINT
valueFrom:
secretKeyRef:
name: trade-dlob-rpc
key: ENDPOINT
- name: WS_ENDPOINT
valueFrom:
secretKeyRef:
name: trade-dlob-rpc
key: WS_ENDPOINT
command: ["node", "/lib/serverLite.js"]
readinessProbe:
httpGet:
path: /startup
port: http
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
httpGet:
path: /health
port: http
initialDelaySeconds: 20
periodSeconds: 20

View File

@@ -0,0 +1,13 @@
apiVersion: v1
kind: Service
metadata:
name: dlob-server
annotations:
argocd.argoproj.io/sync-wave: "4"
spec:
selector:
app.kubernetes.io/name: dlob-server
ports:
- name: http
port: 6969
targetPort: http

View File

@@ -16,7 +16,7 @@ spec:
- name: gitea-registry
containers:
- name: frontend
image: rv32i.pl/trade/trade-frontend:sha-e20a1f5
image: gitea.mpabi.pl/trade/trade-frontend:sha-b06fe7f
imagePullPolicy: IfNotPresent
ports:
- name: http

View File

@@ -37,6 +37,8 @@ spec:
value: "false"
- name: HASURA_GRAPHQL_CORS_DOMAIN
value: "http://localhost:5173,http://127.0.0.1:5173"
- name: HASURA_GRAPHQL_UNAUTHORIZED_ROLE
value: "public"
readinessProbe:
httpGet:
path: /healthz

View File

@@ -51,7 +51,7 @@ function isAlreadyExistsError(errText) {
function isMissingRelationError(errText) {
const t = String(errText || '').toLowerCase();
return t.includes('does not exist') || t.includes('not found') || t.includes('not tracked');
return t.includes('does not exist') || t.includes('not found') || t.includes('not tracked') || t.includes('already untracked');
}
function normalizeName(name) {
@@ -89,10 +89,22 @@ async function main() {
console.log(`[hasura-bootstrap] HASURA_URL=${HASURA_URL}`);
await waitForHasura();
const PUBLIC_DLOB_SOURCE_HEADER = 'X-Hasura-Dlob-Source';
const apiTokensTable = { schema: 'public', name: 'api_tokens' };
const source = 'default';
const baseTicks = { schema: 'public', name: 'drift_ticks' };
const dlobL2LatestTable = { schema: 'public', name: 'dlob_l2_latest' };
const dlobStatsLatestTable = { schema: 'public', name: 'dlob_stats_latest' };
const dlobDepthBpsLatestTable = { schema: 'public', name: 'dlob_depth_bps_latest' };
const dlobSlippageLatestTable = { schema: 'public', name: 'dlob_slippage_latest' };
const dlobSlippageLatestV2Table = { schema: 'public', name: 'dlob_slippage_latest_v2' };
const candlesCacheTable = { schema: 'public', name: 'drift_candles_cache' };
const dlobStatsTsTable = { schema: 'public', name: 'dlob_stats_ts' };
const dlobDepthBpsTsTable = { schema: 'public', name: 'dlob_depth_bps_ts' };
const dlobSlippageTsTable = { schema: 'public', name: 'dlob_slippage_ts' };
const dlobSlippageTsV2Table = { schema: 'public', name: 'dlob_slippage_ts_v2' };
const baseCandlesFn = { schema: 'public', name: 'get_drift_candles' };
const candlesReturnTable = { schema: 'public', name: 'drift_candles' };
@@ -161,8 +173,296 @@ async function main() {
for (const t of tickTables) {
await ensureTickTable(t);
}
// Cached candles table (precomputed by worker; public read).
await ensurePublicSelectTable(candlesCacheTable, [
'bucket',
'bucket_seconds',
'symbol',
'source',
'open',
'high',
'low',
'close',
'oracle_close',
'ticks',
'updated_at',
]);
const ensureDlobTable = async (table, columns, { publicFilter } = {}) => {
await metadataIgnore({ type: 'pg_untrack_table', args: { source, table } });
await metadata({ type: 'pg_track_table', args: { source, table } });
await metadataIgnore({ type: 'pg_drop_select_permission', args: { source, table, role: 'public' } });
await metadata({
type: 'pg_create_select_permission',
args: {
source,
table,
role: 'public',
permission: {
columns,
filter: publicFilter || {},
},
},
});
await metadataIgnore({ type: 'pg_drop_insert_permission', args: { source, table, role: 'ingestor' } });
await metadata({
type: 'pg_create_insert_permission',
args: {
source,
table,
role: 'ingestor',
permission: {
check: {},
set: {},
columns,
},
},
});
await metadataIgnore({ type: 'pg_drop_update_permission', args: { source, table, role: 'ingestor' } });
await metadata({
type: 'pg_create_update_permission',
args: {
source,
table,
role: 'ingestor',
permission: {
filter: {},
check: {},
columns,
},
},
});
};
async function ensurePublicSelectTable(table, columns, { publicFilter } = {}) {
await metadataIgnore({ type: 'pg_untrack_table', args: { source, table } });
await metadata({ type: 'pg_track_table', args: { source, table } });
await metadataIgnore({ type: 'pg_drop_select_permission', args: { source, table, role: 'public' } });
await metadata({
type: 'pg_create_select_permission',
args: {
source,
table,
role: 'public',
permission: {
columns,
filter: publicFilter || {},
},
},
});
// Computed/archived tables are written by workers using admin secret; keep ingestor off by default.
await metadataIgnore({ type: 'pg_drop_insert_permission', args: { source, table, role: 'ingestor' } });
await metadataIgnore({ type: 'pg_drop_update_permission', args: { source, table, role: 'ingestor' } });
}
const dlobPublicFilter = { source: { _eq: PUBLIC_DLOB_SOURCE_HEADER } };
await ensureDlobTable(dlobL2LatestTable, [
'source',
'market_name',
'market_type',
'market_index',
'ts',
'slot',
'mark_price',
'oracle_price',
'best_bid_price',
'best_ask_price',
'bids',
'asks',
'raw',
'updated_at',
], { publicFilter: dlobPublicFilter });
await ensureDlobTable(dlobStatsLatestTable, [
'source',
'market_name',
'market_type',
'market_index',
'ts',
'slot',
'mark_price',
'oracle_price',
'best_bid_price',
'best_ask_price',
'mid_price',
'spread_abs',
'spread_bps',
'depth_levels',
'depth_bid_base',
'depth_ask_base',
'depth_bid_usd',
'depth_ask_usd',
'imbalance',
'raw',
'updated_at',
], { publicFilter: dlobPublicFilter });
await ensurePublicSelectTable(dlobDepthBpsLatestTable, [
'source',
'market_name',
'band_bps',
'market_type',
'market_index',
'ts',
'slot',
'mid_price',
'best_bid_price',
'best_ask_price',
'bid_base',
'ask_base',
'bid_usd',
'ask_usd',
'imbalance',
'raw',
'updated_at',
], { publicFilter: dlobPublicFilter });
await ensurePublicSelectTable(dlobSlippageLatestTable, [
'source',
'market_name',
'side',
'size_usd',
'market_type',
'market_index',
'ts',
'slot',
'mid_price',
'best_bid_price',
'best_ask_price',
'vwap_price',
'worst_price',
'filled_usd',
'filled_base',
'impact_bps',
'levels_consumed',
'fill_pct',
'raw',
'updated_at',
], { publicFilter: dlobPublicFilter });
await ensurePublicSelectTable(dlobSlippageLatestV2Table, [
'source',
'market_name',
'side',
'size_usd',
'market_type',
'market_index',
'ts',
'slot',
'mid_price',
'best_bid_price',
'best_ask_price',
'vwap_price',
'worst_price',
'filled_usd',
'filled_base',
'impact_bps',
'levels_consumed',
'fill_pct',
'raw',
'updated_at',
], { publicFilter: dlobPublicFilter });
await ensurePublicSelectTable(dlobStatsTsTable, [
'ts',
'id',
'source',
'market_name',
'market_type',
'market_index',
'source_ts',
'slot',
'mark_price',
'oracle_price',
'best_bid_price',
'best_ask_price',
'mid_price',
'spread_abs',
'spread_bps',
'depth_levels',
'depth_bid_base',
'depth_ask_base',
'depth_bid_usd',
'depth_ask_usd',
'imbalance',
'raw',
], { publicFilter: dlobPublicFilter });
await ensurePublicSelectTable(dlobDepthBpsTsTable, [
'ts',
'id',
'source',
'market_name',
'band_bps',
'market_type',
'market_index',
'source_ts',
'slot',
'mid_price',
'best_bid_price',
'best_ask_price',
'bid_base',
'ask_base',
'bid_usd',
'ask_usd',
'imbalance',
'raw',
], { publicFilter: dlobPublicFilter });
await ensurePublicSelectTable(dlobSlippageTsTable, [
'ts',
'id',
'source',
'market_name',
'side',
'size_usd',
'market_type',
'market_index',
'source_ts',
'slot',
'mid_price',
'vwap_price',
'worst_price',
'filled_usd',
'filled_base',
'impact_bps',
'levels_consumed',
'fill_pct',
'raw',
], { publicFilter: dlobPublicFilter });
await ensurePublicSelectTable(dlobSlippageTsV2Table, [
'ts',
'id',
'source',
'market_name',
'side',
'size_usd',
'market_type',
'market_index',
'source_ts',
'slot',
'mid_price',
'vwap_price',
'worst_price',
'filled_usd',
'filled_base',
'impact_bps',
'levels_consumed',
'fill_pct',
'raw',
], { publicFilter: dlobPublicFilter });
// Return table type for candle functions (needed for Hasura to track the function).
await metadataIgnore({ type: 'pg_track_table', args: { source, table: candlesReturnTable } });
try {
await metadata({ type: 'pg_track_table', args: { source, table: apiTokensTable } });
} catch (err) {

View File

@@ -3,7 +3,9 @@ kind: Job
metadata:
name: hasura-bootstrap
annotations:
argocd.argoproj.io/sync-wave: "2"
argocd.argoproj.io/hook: Sync
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded
argocd.argoproj.io/sync-wave: "3"
spec:
backoffLimit: 5
template:

View File

@@ -16,7 +16,7 @@ spec:
- name: gitea-registry
containers:
- name: ingestor
image: rv32i.pl/trade/trade-ingestor:k3s-20260106013603
image: gitea.mpabi.pl/trade/trade-ingestor:k3s-20260106013603
imagePullPolicy: IfNotPresent
env:
- name: MARKET_NAME

View File

@@ -121,10 +121,47 @@ CREATE TABLE IF NOT EXISTS public.drift_candles (
high numeric,
low numeric,
close numeric,
oracle_open numeric,
oracle_high numeric,
oracle_low numeric,
oracle_close numeric,
ticks bigint
);
ALTER TABLE public.drift_candles ADD COLUMN IF NOT EXISTS oracle_open numeric;
ALTER TABLE public.drift_candles ADD COLUMN IF NOT EXISTS oracle_high numeric;
ALTER TABLE public.drift_candles ADD COLUMN IF NOT EXISTS oracle_low numeric;
-- Precomputed candle cache (materialized by a worker).
-- Purpose: make tf switching instant by reading ready-made candles instead of aggregating `drift_ticks` on demand.
-- NOTE: `source=''` means "any source" (no source filter).
CREATE TABLE IF NOT EXISTS public.drift_candles_cache (
bucket timestamptz NOT NULL,
bucket_seconds integer NOT NULL,
symbol text NOT NULL,
source text NOT NULL DEFAULT '',
open numeric NOT NULL,
high numeric NOT NULL,
low numeric NOT NULL,
close numeric NOT NULL,
oracle_open numeric,
oracle_high numeric,
oracle_low numeric,
oracle_close numeric,
ticks bigint NOT NULL DEFAULT 0,
updated_at timestamptz NOT NULL DEFAULT now(),
PRIMARY KEY (bucket, bucket_seconds, symbol, source)
);
ALTER TABLE public.drift_candles_cache ADD COLUMN IF NOT EXISTS oracle_open numeric;
ALTER TABLE public.drift_candles_cache ADD COLUMN IF NOT EXISTS oracle_high numeric;
ALTER TABLE public.drift_candles_cache ADD COLUMN IF NOT EXISTS oracle_low numeric;
SELECT create_hypertable('drift_candles_cache', 'bucket', if_not_exists => TRUE, migrate_data => TRUE);
CREATE INDEX IF NOT EXISTS drift_candles_cache_symbol_source_bucket_idx
ON public.drift_candles_cache (symbol, source, bucket_seconds, bucket DESC);
-- If an older version of the function exists with an incompatible return type,
-- CREATE OR REPLACE will fail. Drop the old signature first (safe/idempotent).
DROP FUNCTION IF EXISTS public.get_drift_candles(text, integer, integer, text);
@@ -139,27 +176,641 @@ RETURNS SETOF public.drift_candles
LANGUAGE sql
STABLE
AS $$
WITH base AS (
-- Zwraca zawsze "ciągłe" buckety (fill forward), nawet jeśli nie było ticków w danej sekundzie/minucie.
-- Dzięki temu frontend może rysować regularną oś czasu (np. 1px = 1s) bez dziwnych przeskoków.
WITH src AS (
SELECT COALESCE(p_source, '') AS source_key
),
raw_cached AS (
SELECT
c.bucket,
c.open,
c.high,
c.low,
c.close,
c.oracle_open,
c.oracle_high,
c.oracle_low,
c.oracle_close,
c.ticks
FROM public.drift_candles_cache c, src
WHERE c.symbol = p_symbol
AND c.bucket_seconds = p_bucket_seconds
AND c.source = src.source_key
ORDER BY c.bucket DESC
LIMIT p_limit
),
raw_fallback AS (
SELECT
time_bucket(make_interval(secs => p_bucket_seconds), ts) AS bucket,
ts,
COALESCE(mark_price, oracle_price) AS px,
oracle_price AS oracle_px
FROM public.drift_ticks
FROM public.drift_ticks, src
WHERE symbol = p_symbol
AND (p_source IS NULL OR source = p_source)
AND (src.source_key = '' OR source = src.source_key)
AND ts >= now() - make_interval(secs => (p_bucket_seconds * p_limit * 2))
),
computed AS (
SELECT
bucket,
(array_agg(px ORDER BY ts ASC))[1] AS open,
max(px) AS high,
min(px) AS low,
(array_agg(px ORDER BY ts DESC))[1] AS close,
(array_agg(oracle_px ORDER BY ts ASC))[1] AS oracle_open,
max(oracle_px) AS oracle_high,
min(oracle_px) AS oracle_low,
(array_agg(oracle_px ORDER BY ts DESC))[1] AS oracle_close,
count(*) AS ticks
FROM raw_fallback
GROUP BY bucket
),
data AS (
SELECT * FROM raw_cached
UNION ALL
SELECT * FROM computed
WHERE NOT EXISTS (SELECT 1 FROM raw_cached)
),
bounds AS (
SELECT max(bucket) AS end_bucket FROM data
),
params AS (
SELECT
make_interval(secs => p_bucket_seconds) AS step,
make_interval(secs => (p_bucket_seconds * (p_limit - 1))) AS span
),
series AS (
SELECT generate_series(
bounds.end_bucket - params.span,
bounds.end_bucket,
params.step
) AS bucket
FROM bounds, params
WHERE bounds.end_bucket IS NOT NULL
),
joined AS (
SELECT
s.bucket,
d.open,
d.high,
d.low,
d.close,
d.oracle_open,
d.oracle_high,
d.oracle_low,
d.oracle_close,
d.ticks
FROM series s
LEFT JOIN data d USING (bucket)
ORDER BY s.bucket ASC
),
grouped AS (
SELECT
*,
sum(CASE WHEN close IS NOT NULL THEN 1 ELSE 0 END) OVER (ORDER BY bucket ASC) AS grp_close,
sum(CASE WHEN oracle_close IS NOT NULL THEN 1 ELSE 0 END) OVER (ORDER BY bucket ASC) AS grp_oracle
FROM joined
),
first_vals AS (
SELECT
(SELECT close FROM grouped WHERE close IS NOT NULL ORDER BY bucket ASC LIMIT 1) AS first_close,
(SELECT oracle_close FROM grouped WHERE oracle_close IS NOT NULL ORDER BY bucket ASC LIMIT 1) AS first_oracle
),
ff AS (
SELECT
g.bucket,
g.open,
g.high,
g.low,
g.close,
g.oracle_open,
g.oracle_high,
g.oracle_low,
g.oracle_close,
g.ticks,
COALESCE(
g.close,
max(g.close) OVER (PARTITION BY g.grp_close),
f.first_close
) AS ff_close,
COALESCE(
g.oracle_close,
max(g.oracle_close) OVER (PARTITION BY g.grp_oracle),
f.first_oracle
) AS ff_oracle
FROM grouped g
CROSS JOIN first_vals f
)
SELECT
bucket,
(array_agg(px ORDER BY ts ASC))[1] AS open,
max(px) AS high,
min(px) AS low,
(array_agg(px ORDER BY ts DESC))[1] AS close,
(array_agg(oracle_px ORDER BY ts DESC))[1] AS oracle_close,
count(*) AS ticks
FROM base
GROUP BY bucket
COALESCE(open, ff_close) AS open,
COALESCE(high, ff_close) AS high,
COALESCE(low, ff_close) AS low,
COALESCE(close, ff_close) AS close,
COALESCE(oracle_open, ff_oracle) AS oracle_open,
COALESCE(oracle_high, ff_oracle) AS oracle_high,
COALESCE(oracle_low, ff_oracle) AS oracle_low,
COALESCE(oracle_close, ff_oracle) AS oracle_close,
COALESCE(ticks, 0) AS ticks
FROM ff
ORDER BY bucket DESC
LIMIT p_limit;
$$;
-- Latest DLOB orderbook snapshots (top-N levels), per market.
-- Filled by a VPS worker (collector) and consumed by the UI via Hasura subscriptions.
CREATE TABLE IF NOT EXISTS public.dlob_l2_latest (
source TEXT NOT NULL DEFAULT 'mevnode',
market_name TEXT NOT NULL,
market_type TEXT NOT NULL DEFAULT 'perp',
market_index INTEGER,
ts BIGINT,
slot BIGINT,
mark_price NUMERIC,
oracle_price NUMERIC,
best_bid_price NUMERIC,
best_ask_price NUMERIC,
bids JSONB,
asks JSONB,
raw JSONB,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (source, market_name)
);
-- Schema upgrades (idempotent for existing volumes)
ALTER TABLE public.dlob_l2_latest ADD COLUMN IF NOT EXISTS source TEXT;
ALTER TABLE public.dlob_l2_latest ALTER COLUMN source SET DEFAULT 'mevnode';
UPDATE public.dlob_l2_latest SET source = 'mevnode' WHERE source IS NULL;
ALTER TABLE public.dlob_l2_latest ALTER COLUMN source SET NOT NULL;
-- Ensure PRIMARY KEY is (source, market_name) (required to keep 2 sources in parallel).
DO $$
DECLARE
pk_name text;
pk_cols text[];
BEGIN
SELECT
con.conname,
array_agg(att.attname ORDER BY ord.ordinality)
INTO pk_name, pk_cols
FROM pg_constraint con
JOIN pg_class rel ON rel.oid = con.conrelid
JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
JOIN unnest(con.conkey) WITH ORDINALITY AS ord(attnum, ordinality) ON true
JOIN pg_attribute att ON att.attrelid = rel.oid AND att.attnum = ord.attnum
WHERE con.contype = 'p' AND nsp.nspname = 'public' AND rel.relname = 'dlob_l2_latest'
GROUP BY con.conname;
IF pk_name IS NULL THEN
EXECUTE 'ALTER TABLE public.dlob_l2_latest ADD CONSTRAINT dlob_l2_latest_pkey PRIMARY KEY (source, market_name)';
ELSIF pk_cols <> ARRAY['source','market_name'] THEN
EXECUTE format('ALTER TABLE public.dlob_l2_latest DROP CONSTRAINT %I', pk_name);
EXECUTE 'ALTER TABLE public.dlob_l2_latest ADD CONSTRAINT dlob_l2_latest_pkey PRIMARY KEY (source, market_name)';
END IF;
END $$;
CREATE INDEX IF NOT EXISTS dlob_l2_latest_updated_at_idx
ON public.dlob_l2_latest (updated_at DESC);
CREATE INDEX IF NOT EXISTS dlob_l2_latest_source_updated_at_idx
ON public.dlob_l2_latest (source, updated_at DESC);
-- Derived stats for fast UI display.
CREATE TABLE IF NOT EXISTS public.dlob_stats_latest (
source TEXT NOT NULL DEFAULT 'mevnode',
market_name TEXT NOT NULL,
market_type TEXT NOT NULL DEFAULT 'perp',
market_index INTEGER,
ts BIGINT,
slot BIGINT,
mark_price NUMERIC,
oracle_price NUMERIC,
best_bid_price NUMERIC,
best_ask_price NUMERIC,
mid_price NUMERIC,
spread_abs NUMERIC,
spread_bps NUMERIC,
depth_levels INTEGER,
depth_bid_base NUMERIC,
depth_ask_base NUMERIC,
depth_bid_usd NUMERIC,
depth_ask_usd NUMERIC,
imbalance NUMERIC,
raw JSONB,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (source, market_name)
);
-- Schema upgrades (idempotent for existing volumes)
ALTER TABLE public.dlob_stats_latest ADD COLUMN IF NOT EXISTS source TEXT;
ALTER TABLE public.dlob_stats_latest ALTER COLUMN source SET DEFAULT 'mevnode';
UPDATE public.dlob_stats_latest SET source = 'mevnode' WHERE source IS NULL;
ALTER TABLE public.dlob_stats_latest ALTER COLUMN source SET NOT NULL;
-- Ensure PRIMARY KEY is (source, market_name) (required to keep 2 sources in parallel).
DO $$
DECLARE
pk_name text;
pk_cols text[];
BEGIN
SELECT
con.conname,
array_agg(att.attname ORDER BY ord.ordinality)
INTO pk_name, pk_cols
FROM pg_constraint con
JOIN pg_class rel ON rel.oid = con.conrelid
JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
JOIN unnest(con.conkey) WITH ORDINALITY AS ord(attnum, ordinality) ON true
JOIN pg_attribute att ON att.attrelid = rel.oid AND att.attnum = ord.attnum
WHERE con.contype = 'p' AND nsp.nspname = 'public' AND rel.relname = 'dlob_stats_latest'
GROUP BY con.conname;
IF pk_name IS NULL THEN
EXECUTE 'ALTER TABLE public.dlob_stats_latest ADD CONSTRAINT dlob_stats_latest_pkey PRIMARY KEY (source, market_name)';
ELSIF pk_cols <> ARRAY['source','market_name'] THEN
EXECUTE format('ALTER TABLE public.dlob_stats_latest DROP CONSTRAINT %I', pk_name);
EXECUTE 'ALTER TABLE public.dlob_stats_latest ADD CONSTRAINT dlob_stats_latest_pkey PRIMARY KEY (source, market_name)';
END IF;
END $$;
CREATE INDEX IF NOT EXISTS dlob_stats_latest_updated_at_idx
ON public.dlob_stats_latest (updated_at DESC);
CREATE INDEX IF NOT EXISTS dlob_stats_latest_source_updated_at_idx
ON public.dlob_stats_latest (source, updated_at DESC);
-- Depth snapshots within bps bands around mid-price (per market, per band).
-- Filled by a derived worker that reads `dlob_l2_latest`.
CREATE TABLE IF NOT EXISTS public.dlob_depth_bps_latest (
source TEXT NOT NULL DEFAULT 'mevnode',
market_name TEXT NOT NULL,
band_bps INTEGER NOT NULL,
market_type TEXT NOT NULL DEFAULT 'perp',
market_index INTEGER,
ts BIGINT,
slot BIGINT,
mid_price NUMERIC,
best_bid_price NUMERIC,
best_ask_price NUMERIC,
bid_base NUMERIC,
ask_base NUMERIC,
bid_usd NUMERIC,
ask_usd NUMERIC,
imbalance NUMERIC,
raw JSONB,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (source, market_name, band_bps)
);
-- Schema upgrades (idempotent for existing volumes)
ALTER TABLE public.dlob_depth_bps_latest ADD COLUMN IF NOT EXISTS source TEXT;
ALTER TABLE public.dlob_depth_bps_latest ALTER COLUMN source SET DEFAULT 'mevnode';
UPDATE public.dlob_depth_bps_latest SET source = 'mevnode' WHERE source IS NULL;
ALTER TABLE public.dlob_depth_bps_latest ALTER COLUMN source SET NOT NULL;
-- Ensure PRIMARY KEY is (source, market_name, band_bps) (required to keep 2 sources in parallel).
DO $$
DECLARE
pk_name text;
pk_cols text[];
BEGIN
SELECT
con.conname,
array_agg(att.attname ORDER BY ord.ordinality)
INTO pk_name, pk_cols
FROM pg_constraint con
JOIN pg_class rel ON rel.oid = con.conrelid
JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
JOIN unnest(con.conkey) WITH ORDINALITY AS ord(attnum, ordinality) ON true
JOIN pg_attribute att ON att.attrelid = rel.oid AND att.attnum = ord.attnum
WHERE con.contype = 'p' AND nsp.nspname = 'public' AND rel.relname = 'dlob_depth_bps_latest'
GROUP BY con.conname;
IF pk_name IS NULL THEN
EXECUTE 'ALTER TABLE public.dlob_depth_bps_latest ADD CONSTRAINT dlob_depth_bps_latest_pkey PRIMARY KEY (source, market_name, band_bps)';
ELSIF pk_cols <> ARRAY['source','market_name','band_bps'] THEN
EXECUTE format('ALTER TABLE public.dlob_depth_bps_latest DROP CONSTRAINT %I', pk_name);
EXECUTE 'ALTER TABLE public.dlob_depth_bps_latest ADD CONSTRAINT dlob_depth_bps_latest_pkey PRIMARY KEY (source, market_name, band_bps)';
END IF;
END $$;
CREATE INDEX IF NOT EXISTS dlob_depth_bps_latest_updated_at_idx
ON public.dlob_depth_bps_latest (updated_at DESC);
CREATE INDEX IF NOT EXISTS dlob_depth_bps_latest_market_name_idx
ON public.dlob_depth_bps_latest (market_name);
CREATE INDEX IF NOT EXISTS dlob_depth_bps_latest_source_market_name_idx
ON public.dlob_depth_bps_latest (source, market_name);
-- Slippage/impact estimates for "market" orders at common USD sizes.
-- Filled by a derived worker that reads `dlob_l2_latest`.
CREATE TABLE IF NOT EXISTS public.dlob_slippage_latest (
source TEXT NOT NULL DEFAULT 'mevnode',
market_name TEXT NOT NULL,
side TEXT NOT NULL,
size_usd INTEGER NOT NULL,
market_type TEXT NOT NULL DEFAULT 'perp',
market_index INTEGER,
ts BIGINT,
slot BIGINT,
mid_price NUMERIC,
best_bid_price NUMERIC,
best_ask_price NUMERIC,
vwap_price NUMERIC,
worst_price NUMERIC,
filled_usd NUMERIC,
filled_base NUMERIC,
impact_bps NUMERIC,
levels_consumed INTEGER,
fill_pct NUMERIC,
raw JSONB,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (source, market_name, side, size_usd),
CONSTRAINT dlob_slippage_latest_side_chk CHECK (side IN ('buy', 'sell'))
);
-- Schema upgrades (idempotent for existing volumes)
ALTER TABLE public.dlob_slippage_latest ADD COLUMN IF NOT EXISTS source TEXT;
ALTER TABLE public.dlob_slippage_latest ALTER COLUMN source SET DEFAULT 'mevnode';
UPDATE public.dlob_slippage_latest SET source = 'mevnode' WHERE source IS NULL;
ALTER TABLE public.dlob_slippage_latest ALTER COLUMN source SET NOT NULL;
-- Ensure PRIMARY KEY is (source, market_name, side, size_usd) (required to keep 2 sources in parallel).
DO $$
DECLARE
pk_name text;
pk_cols text[];
BEGIN
SELECT
con.conname,
array_agg(att.attname ORDER BY ord.ordinality)
INTO pk_name, pk_cols
FROM pg_constraint con
JOIN pg_class rel ON rel.oid = con.conrelid
JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
JOIN unnest(con.conkey) WITH ORDINALITY AS ord(attnum, ordinality) ON true
JOIN pg_attribute att ON att.attrelid = rel.oid AND att.attnum = ord.attnum
WHERE con.contype = 'p' AND nsp.nspname = 'public' AND rel.relname = 'dlob_slippage_latest'
GROUP BY con.conname;
IF pk_name IS NULL THEN
EXECUTE 'ALTER TABLE public.dlob_slippage_latest ADD CONSTRAINT dlob_slippage_latest_pkey PRIMARY KEY (source, market_name, side, size_usd)';
ELSIF pk_cols <> ARRAY['source','market_name','side','size_usd'] THEN
EXECUTE format('ALTER TABLE public.dlob_slippage_latest DROP CONSTRAINT %I', pk_name);
EXECUTE 'ALTER TABLE public.dlob_slippage_latest ADD CONSTRAINT dlob_slippage_latest_pkey PRIMARY KEY (source, market_name, side, size_usd)';
END IF;
END $$;
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_updated_at_idx
ON public.dlob_slippage_latest (updated_at DESC);
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_market_name_idx
ON public.dlob_slippage_latest (market_name);
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_source_market_name_idx
ON public.dlob_slippage_latest (source, market_name);
-- Slippage v2: supports fractional order sizes (e.g. 0.1/0.2/0.5 USD), per market and side.
-- Keep v1 intact for backward compatibility and to avoid data loss.
CREATE TABLE IF NOT EXISTS public.dlob_slippage_latest_v2 (
source TEXT NOT NULL DEFAULT 'mevnode',
market_name TEXT NOT NULL,
side TEXT NOT NULL, -- buy|sell
size_usd NUMERIC NOT NULL,
market_type TEXT NOT NULL DEFAULT 'perp',
market_index INTEGER,
ts BIGINT,
slot BIGINT,
mid_price NUMERIC,
best_bid_price NUMERIC,
best_ask_price NUMERIC,
vwap_price NUMERIC,
worst_price NUMERIC,
filled_usd NUMERIC,
filled_base NUMERIC,
impact_bps NUMERIC,
levels_consumed INTEGER,
fill_pct NUMERIC,
raw JSONB,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (source, market_name, side, size_usd),
CONSTRAINT dlob_slippage_latest_v2_side_chk CHECK (side IN ('buy', 'sell'))
);
-- Schema upgrades (idempotent for existing volumes)
ALTER TABLE public.dlob_slippage_latest_v2 ADD COLUMN IF NOT EXISTS source TEXT;
ALTER TABLE public.dlob_slippage_latest_v2 ALTER COLUMN source SET DEFAULT 'mevnode';
UPDATE public.dlob_slippage_latest_v2 SET source = 'mevnode' WHERE source IS NULL;
ALTER TABLE public.dlob_slippage_latest_v2 ALTER COLUMN source SET NOT NULL;
-- Ensure PRIMARY KEY is (source, market_name, side, size_usd) (required to keep 2 sources in parallel).
DO $$
DECLARE
pk_name text;
pk_cols text[];
BEGIN
SELECT
con.conname,
array_agg(att.attname ORDER BY ord.ordinality)
INTO pk_name, pk_cols
FROM pg_constraint con
JOIN pg_class rel ON rel.oid = con.conrelid
JOIN pg_namespace nsp ON nsp.oid = rel.relnamespace
JOIN unnest(con.conkey) WITH ORDINALITY AS ord(attnum, ordinality) ON true
JOIN pg_attribute att ON att.attrelid = rel.oid AND att.attnum = ord.attnum
WHERE con.contype = 'p' AND nsp.nspname = 'public' AND rel.relname = 'dlob_slippage_latest_v2'
GROUP BY con.conname;
IF pk_name IS NULL THEN
EXECUTE 'ALTER TABLE public.dlob_slippage_latest_v2 ADD CONSTRAINT dlob_slippage_latest_v2_pkey PRIMARY KEY (source, market_name, side, size_usd)';
ELSIF pk_cols <> ARRAY['source','market_name','side','size_usd'] THEN
EXECUTE format('ALTER TABLE public.dlob_slippage_latest_v2 DROP CONSTRAINT %I', pk_name);
EXECUTE 'ALTER TABLE public.dlob_slippage_latest_v2 ADD CONSTRAINT dlob_slippage_latest_v2_pkey PRIMARY KEY (source, market_name, side, size_usd)';
END IF;
END $$;
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_v2_updated_at_idx
ON public.dlob_slippage_latest_v2 (updated_at DESC);
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_v2_market_name_idx
ON public.dlob_slippage_latest_v2 (market_name);
CREATE INDEX IF NOT EXISTS dlob_slippage_latest_v2_source_market_name_idx
ON public.dlob_slippage_latest_v2 (source, market_name);
-- Time-series tables for UI history (start: 7 days).
-- Keep these append-only; use Timescale hypertables.
CREATE TABLE IF NOT EXISTS public.dlob_stats_ts (
ts TIMESTAMPTZ NOT NULL,
id BIGSERIAL NOT NULL,
source TEXT NOT NULL DEFAULT 'mevnode',
market_name TEXT NOT NULL,
market_type TEXT NOT NULL DEFAULT 'perp',
market_index INTEGER,
source_ts BIGINT,
slot BIGINT,
mark_price NUMERIC,
oracle_price NUMERIC,
best_bid_price NUMERIC,
best_ask_price NUMERIC,
mid_price NUMERIC,
spread_abs NUMERIC,
spread_bps NUMERIC,
depth_levels INTEGER,
depth_bid_base NUMERIC,
depth_ask_base NUMERIC,
depth_bid_usd NUMERIC,
depth_ask_usd NUMERIC,
imbalance NUMERIC,
raw JSONB,
PRIMARY KEY (ts, id)
);
-- Schema upgrades (idempotent for existing volumes)
ALTER TABLE public.dlob_stats_ts ADD COLUMN IF NOT EXISTS source TEXT;
ALTER TABLE public.dlob_stats_ts ALTER COLUMN source SET DEFAULT 'mevnode';
UPDATE public.dlob_stats_ts SET source = 'mevnode' WHERE source IS NULL;
ALTER TABLE public.dlob_stats_ts ALTER COLUMN source SET NOT NULL;
SELECT create_hypertable('dlob_stats_ts', 'ts', if_not_exists => TRUE, migrate_data => TRUE);
CREATE INDEX IF NOT EXISTS dlob_stats_ts_market_ts_desc_idx
ON public.dlob_stats_ts (market_name, ts DESC);
CREATE INDEX IF NOT EXISTS dlob_stats_ts_source_market_ts_desc_idx
ON public.dlob_stats_ts (source, market_name, ts DESC);
CREATE TABLE IF NOT EXISTS public.dlob_depth_bps_ts (
ts TIMESTAMPTZ NOT NULL,
id BIGSERIAL NOT NULL,
source TEXT NOT NULL DEFAULT 'mevnode',
market_name TEXT NOT NULL,
band_bps INTEGER NOT NULL,
market_type TEXT NOT NULL DEFAULT 'perp',
market_index INTEGER,
source_ts BIGINT,
slot BIGINT,
mid_price NUMERIC,
best_bid_price NUMERIC,
best_ask_price NUMERIC,
bid_base NUMERIC,
ask_base NUMERIC,
bid_usd NUMERIC,
ask_usd NUMERIC,
imbalance NUMERIC,
raw JSONB,
PRIMARY KEY (ts, id)
);
-- Schema upgrades (idempotent for existing volumes)
ALTER TABLE public.dlob_depth_bps_ts ADD COLUMN IF NOT EXISTS source TEXT;
ALTER TABLE public.dlob_depth_bps_ts ALTER COLUMN source SET DEFAULT 'mevnode';
UPDATE public.dlob_depth_bps_ts SET source = 'mevnode' WHERE source IS NULL;
ALTER TABLE public.dlob_depth_bps_ts ALTER COLUMN source SET NOT NULL;
SELECT create_hypertable('dlob_depth_bps_ts', 'ts', if_not_exists => TRUE, migrate_data => TRUE);
CREATE INDEX IF NOT EXISTS dlob_depth_bps_ts_market_ts_desc_idx
ON public.dlob_depth_bps_ts (market_name, ts DESC);
CREATE INDEX IF NOT EXISTS dlob_depth_bps_ts_source_market_ts_desc_idx
ON public.dlob_depth_bps_ts (source, market_name, ts DESC);
CREATE TABLE IF NOT EXISTS public.dlob_slippage_ts (
ts TIMESTAMPTZ NOT NULL,
id BIGSERIAL NOT NULL,
source TEXT NOT NULL DEFAULT 'mevnode',
market_name TEXT NOT NULL,
side TEXT NOT NULL,
size_usd INTEGER NOT NULL,
market_type TEXT NOT NULL DEFAULT 'perp',
market_index INTEGER,
source_ts BIGINT,
slot BIGINT,
mid_price NUMERIC,
vwap_price NUMERIC,
worst_price NUMERIC,
filled_usd NUMERIC,
filled_base NUMERIC,
impact_bps NUMERIC,
levels_consumed INTEGER,
fill_pct NUMERIC,
raw JSONB,
PRIMARY KEY (ts, id)
);
-- Schema upgrades (idempotent for existing volumes)
ALTER TABLE public.dlob_slippage_ts ADD COLUMN IF NOT EXISTS source TEXT;
ALTER TABLE public.dlob_slippage_ts ALTER COLUMN source SET DEFAULT 'mevnode';
UPDATE public.dlob_slippage_ts SET source = 'mevnode' WHERE source IS NULL;
ALTER TABLE public.dlob_slippage_ts ALTER COLUMN source SET NOT NULL;
SELECT create_hypertable('dlob_slippage_ts', 'ts', if_not_exists => TRUE, migrate_data => TRUE);
CREATE INDEX IF NOT EXISTS dlob_slippage_ts_market_ts_desc_idx
ON public.dlob_slippage_ts (market_name, ts DESC);
CREATE INDEX IF NOT EXISTS dlob_slippage_ts_source_market_ts_desc_idx
ON public.dlob_slippage_ts (source, market_name, ts DESC);
CREATE TABLE IF NOT EXISTS public.dlob_slippage_ts_v2 (
ts TIMESTAMPTZ NOT NULL,
id BIGSERIAL NOT NULL,
source TEXT NOT NULL DEFAULT 'mevnode',
market_name TEXT NOT NULL,
side TEXT NOT NULL,
size_usd NUMERIC NOT NULL,
market_type TEXT NOT NULL DEFAULT 'perp',
market_index INTEGER,
source_ts BIGINT,
slot BIGINT,
mid_price NUMERIC,
vwap_price NUMERIC,
worst_price NUMERIC,
filled_usd NUMERIC,
filled_base NUMERIC,
impact_bps NUMERIC,
levels_consumed INTEGER,
fill_pct NUMERIC,
raw JSONB,
PRIMARY KEY (ts, id)
);
-- Schema upgrades (idempotent for existing volumes)
ALTER TABLE public.dlob_slippage_ts_v2 ADD COLUMN IF NOT EXISTS source TEXT;
ALTER TABLE public.dlob_slippage_ts_v2 ALTER COLUMN source SET DEFAULT 'mevnode';
UPDATE public.dlob_slippage_ts_v2 SET source = 'mevnode' WHERE source IS NULL;
ALTER TABLE public.dlob_slippage_ts_v2 ALTER COLUMN source SET NOT NULL;
SELECT create_hypertable('dlob_slippage_ts_v2', 'ts', if_not_exists => TRUE, migrate_data => TRUE);
CREATE INDEX IF NOT EXISTS dlob_slippage_ts_v2_market_ts_desc_idx
ON public.dlob_slippage_ts_v2 (market_name, ts DESC);
CREATE INDEX IF NOT EXISTS dlob_slippage_ts_v2_source_market_ts_desc_idx
ON public.dlob_slippage_ts_v2 (source, market_name, ts DESC);
-- Retention policies (best-effort; safe if Timescale is present).
DO $$
BEGIN
PERFORM add_retention_policy('dlob_stats_ts', INTERVAL '7 days');
EXCEPTION WHEN OTHERS THEN
-- ignore if policy exists or function unavailable
END $$;
DO $$
BEGIN
PERFORM add_retention_policy('dlob_depth_bps_ts', INTERVAL '7 days');
EXCEPTION WHEN OTHERS THEN
END $$;
DO $$
BEGIN
PERFORM add_retention_policy('dlob_slippage_ts', INTERVAL '7 days');
EXCEPTION WHEN OTHERS THEN
END $$;
DO $$
BEGIN
PERFORM add_retention_policy('dlob_slippage_ts_v2', INTERVAL '7 days');
EXCEPTION WHEN OTHERS THEN
END $$;

View File

@@ -5,6 +5,7 @@ resources:
- cm-trade-deploy.yaml
- postgres/service.yaml
- postgres/statefulset.yaml
- postgres/job-migrate.yaml
- hasura/service.yaml
- hasura/deployment.yaml
- hasura/job-bootstrap.yaml
@@ -13,6 +14,19 @@ resources:
- ingestor/deployment.yaml
- frontend/service.yaml
- frontend/deployment.yaml
- dlob/redis.yaml
- dlob/publisher-deployment.yaml
- dlob/server-service.yaml
- dlob/server-deployment.yaml
- dlob-worker/deployment.yaml
- dlob-worker/deployment-drift.yaml
- dlob-depth-worker/deployment.yaml
- dlob-depth-worker/deployment-drift.yaml
- dlob-slippage-worker/deployment.yaml
- dlob-slippage-worker/deployment-drift.yaml
- dlob-ts-archiver/deployment.yaml
- dlob-ts-archiver/deployment-drift.yaml
- candles-cache-worker/deployment.yaml
configMapGenerator:
- name: postgres-initdb
@@ -21,6 +35,27 @@ configMapGenerator:
- name: hasura-bootstrap-script
files:
- hasura/hasura-bootstrap.mjs
- name: dlob-worker-script
files:
- dlob-worker/worker.mjs
- name: dlob-depth-worker-script
files:
- dlob-depth-worker/worker.mjs
- name: dlob-slippage-worker-script
files:
- dlob-slippage-worker/worker.mjs
- name: dlob-ts-archiver-script
files:
- dlob-ts-archiver/worker.mjs
- name: candles-cache-worker-script
files:
- candles-cache-worker/worker.mjs
- name: trade-api-wrapper
files:
- api/wrapper.mjs
- name: trade-api-upstream
files:
- api/server.mjs
generatorOptions:
disableNameSuffixHash: true

View File

@@ -0,0 +1,36 @@
apiVersion: batch/v1
kind: Job
metadata:
name: postgres-migrate
annotations:
argocd.argoproj.io/hook: Sync
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded
argocd.argoproj.io/sync-wave: "2"
spec:
backoffLimit: 3
template:
spec:
restartPolicy: OnFailure
containers:
- name: postgres-migrate
image: postgres:16
imagePullPolicy: IfNotPresent
envFrom:
- secretRef:
name: trade-postgres
command:
- sh
- -ec
- |
export PGPASSWORD="$POSTGRES_PASSWORD"
until pg_isready -h postgres -U "$POSTGRES_USER" -d "$POSTGRES_DB"; do sleep 1; done
psql -h postgres -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 -f /migrations/001_init.sql
volumeMounts:
- name: migrations
mountPath: /migrations/001_init.sql
subPath: 001_init.sql
readOnly: true
volumes:
- name: migrations
configMap:
name: postgres-initdb

View File

@@ -0,0 +1,13 @@
apiVersion: cert-manager.io/v1
kind: Certificate
metadata:
name: monitoring-mpabi-pl
namespace: monitoring
spec:
secretName: monitoring-mpabi-pl-tls
issuerRef:
kind: ClusterIssuer
name: letsencrypt-prod
dnsNames:
- grafana.mpabi.pl
- prometheus.mpabi.pl

View File

@@ -0,0 +1,238 @@
apiVersion: v1
kind: ConfigMap
metadata:
name: grafana-dashboard-agave-status
namespace: monitoring
labels:
grafana_dashboard: "1"
data:
agave-status.json: |-
{
"uid": "agave-status-mpabi",
"title": "Agave @ mpabi",
"timezone": "browser",
"schemaVersion": 39,
"version": 2,
"time": { "from": "now-6h", "to": "now" },
"timepicker": {
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m"],
"time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d"]
},
"refresh": "10s",
"tags": ["agave", "solana", "mpabi"],
"templating": {
"list": [
{
"name": "instance",
"type": "query",
"label": "Node Exporter Instance",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"query": { "query": "label_values(up{job=\"mpabi-node-exporter\"}, instance)", "refId": "PromVarInstance" },
"current": { "selected": false, "text": "10.66.66.1:9100", "value": "10.66.66.1:9100" }
}
]
},
"panels": [
{
"id": 1,
"type": "stat",
"title": "Geyser Metrics Target (Prometheus up)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "A", "expr": "up{job=\"mpabi-yellowstone-geyser\"}" }],
"options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "horizontal", "textMode": "value" },
"fieldConfig": {
"defaults": { "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } },
"overrides": []
},
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 }
},
{
"id": 2,
"type": "stat",
"title": "agave-validator.service (systemd active)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "B", "expr": "node_systemd_unit_state{job=\"mpabi-node-exporter\",instance=\"$instance\",name=\"agave-validator.service\",state=\"active\"}" }],
"options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "horizontal", "textMode": "value" },
"fieldConfig": {
"defaults": { "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } },
"overrides": []
},
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 }
},
{
"id": 3,
"type": "timeseries",
"title": "CPU Used (%)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "C", "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{job=\"mpabi-node-exporter\",instance=\"$instance\",mode=\"idle\"}[5m])) * 100)" }],
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 6 }
},
{
"id": 4,
"type": "timeseries",
"title": "Load (1m)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "D", "expr": "node_load1{job=\"mpabi-node-exporter\",instance=\"$instance\"}" }],
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 6 }
},
{
"id": 5,
"type": "timeseries",
"title": "Memory Used (%)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "E", "expr": "100 - (node_memory_MemAvailable_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\"} / node_memory_MemTotal_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\"} * 100)" }],
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 6 }
},
{
"id": 6,
"type": "timeseries",
"title": "Swap Used (GiB)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "F", "expr": "(node_memory_SwapTotal_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\"} - node_memory_SwapFree_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\"}) / 1024 / 1024 / 1024" }],
"fieldConfig": { "defaults": { "unit": "gbytes" }, "overrides": [] },
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 14 }
},
{
"id": 7,
"type": "timeseries",
"title": "Disk Free Accounts (%)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "G", "expr": "100 * node_filesystem_avail_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\",mountpoint=\"/var/lib/solana/accounts\"} / node_filesystem_size_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\",mountpoint=\"/var/lib/solana/accounts\"}" }],
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 14 }
},
{
"id": 8,
"type": "timeseries",
"title": "Disk Free Ledger (%)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "H", "expr": "100 * node_filesystem_avail_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\",mountpoint=\"/var/lib/solana/ledger\"} / node_filesystem_size_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\",mountpoint=\"/var/lib/solana/ledger\"}" }],
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 14 }
},
{
"id": 9,
"type": "timeseries",
"title": "Disk IO (NVMe) Read/Write (MiB/s)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{ "refId": "I", "expr": "sum by (device) (rate(node_disk_read_bytes_total{job=\"mpabi-node-exporter\",instance=\"$instance\",device=~\"nvme.*\"}[5m])) / 1024 / 1024", "legendFormat": "read {{device}}" },
{ "refId": "J", "expr": "sum by (device) (rate(node_disk_written_bytes_total{job=\"mpabi-node-exporter\",instance=\"$instance\",device=~\"nvme.*\"}[5m])) / 1024 / 1024", "legendFormat": "write {{device}}" }
],
"fieldConfig": { "defaults": { "unit": "mbytes" }, "overrides": [] },
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }
},
{
"id": 10,
"type": "timeseries",
"title": "Network wg0 RX/TX (MiB/s)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{ "refId": "K", "expr": "rate(node_network_receive_bytes_total{job=\"mpabi-node-exporter\",instance=\"$instance\",device=\"wg0\"}[5m]) / 1024 / 1024", "legendFormat": "rx" },
{ "refId": "L", "expr": "rate(node_network_transmit_bytes_total{job=\"mpabi-node-exporter\",instance=\"$instance\",device=\"wg0\"}[5m]) / 1024 / 1024", "legendFormat": "tx" }
],
"fieldConfig": { "defaults": { "unit": "mbytes" }, "overrides": [] },
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }
},
{
"id": 11,
"type": "timeseries",
"title": "Geyser: Subscriber Queue Size",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "M", "expr": "grpc_subscriber_queue_size{job=\"mpabi-yellowstone-geyser\"}", "legendFormat": "{{subscriber_id}}" }],
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 30 }
},
{
"id": 12,
"type": "timeseries",
"title": "Geyser: Connections Total",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "N", "expr": "connections_total{job=\"mpabi-yellowstone-geyser\"}" }],
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 30 }
},
{
"id": 13,
"type": "timeseries",
"title": "Geyser: Bytes Sent (MiB/s)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "O", "expr": "rate(grpc_bytes_sent{job=\"mpabi-yellowstone-geyser\"}[5m]) / 1024 / 1024", "legendFormat": "{{subscriber_id}}" }],
"fieldConfig": { "defaults": { "unit": "mbytes" }, "overrides": [] },
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 }
},
{
"id": 14,
"type": "timeseries",
"title": "Geyser: Messages Sent (/s)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "P", "expr": "rate(grpc_message_sent_count{job=\"mpabi-yellowstone-geyser\"}[5m])", "legendFormat": "{{subscriber_id}}" }],
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 38 }
},
{
"id": 15,
"type": "timeseries",
"title": "Geyser: Disconnects (increase 15m)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "Q", "expr": "sum by (reason) (increase(grpc_client_disconnects_total{job=\"mpabi-yellowstone-geyser\"}[15m]))", "legendFormat": "{{reason}}" }],
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 46 }
},
{
"id": 16,
"type": "stat",
"title": "RPC Slot Lag (slots)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "R", "expr": "solana_rpc_slot_lag{job=\"mpabi-node-exporter\",instance=\"$instance\"}" }],
"fieldConfig": {
"defaults": {
"unit": "short",
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 20 },
{ "color": "red", "value": 50 }
]
}
},
"overrides": []
},
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 54 }
},
{
"id": 17,
"type": "stat",
"title": "RPC Slot Lag (szac. minuty)",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [{ "refId": "S", "expr": "solana_rpc_slot_lag{job=\"mpabi-node-exporter\",instance=\"$instance\"} * 0.4 / 60" }],
"fieldConfig": {
"defaults": {
"unit": "min",
"decimals": 2,
"thresholds": {
"mode": "absolute",
"steps": [
{ "color": "green", "value": null },
{ "color": "yellow", "value": 1 },
{ "color": "red", "value": 2 }
]
}
},
"overrides": []
},
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 54 }
},
{
"id": 18,
"type": "timeseries",
"title": "RPC Slot & Reference Slot",
"datasource": { "type": "prometheus", "uid": "prometheus" },
"targets": [
{ "refId": "T", "expr": "solana_rpc_slot{job=\"mpabi-node-exporter\",instance=\"$instance\"}" },
{ "refId": "U", "expr": "solana_rpc_slot_reference{job=\"mpabi-node-exporter\",instance=\"$instance\"}" },
{ "refId": "V", "expr": "solana_rpc_block_height{job=\"mpabi-node-exporter\",instance=\"$instance\"}" }
],
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 60 }
}
]
}

View File

@@ -0,0 +1,16 @@
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
name: grafana-http
namespace: monitoring
spec:
entryPoints:
- web
routes:
- match: Host(`grafana.mpabi.pl`)
kind: Rule
middlewares:
- name: redirect-to-https
services:
- name: monitoring-stack-grafana
port: 80

View File

@@ -0,0 +1,16 @@
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
name: grafana
namespace: monitoring
spec:
entryPoints:
- websecure
routes:
- match: Host(`grafana.mpabi.pl`)
kind: Rule
services:
- name: monitoring-stack-grafana
port: 80
tls:
secretName: monitoring-mpabi-pl-tls

View File

@@ -0,0 +1,16 @@
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
name: prometheus-http
namespace: monitoring
spec:
entryPoints:
- web
routes:
- match: Host(`prometheus.mpabi.pl`)
kind: Rule
middlewares:
- name: redirect-to-https
services:
- name: monitoring-stack-kube-prom-prometheus
port: 9090

View File

@@ -0,0 +1,16 @@
apiVersion: traefik.io/v1alpha1
kind: IngressRoute
metadata:
name: prometheus
namespace: monitoring
spec:
entryPoints:
- websecure
routes:
- match: Host(`prometheus.mpabi.pl`)
kind: Rule
services:
- name: monitoring-stack-kube-prom-prometheus
port: 9090
tls:
secretName: monitoring-mpabi-pl-tls

View File

@@ -0,0 +1,11 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- middleware-redirect-to-https.yaml
- certificate-monitoring-mpabi-pl.yaml
- ingressroute-grafana.yaml
- ingressroute-grafana-http.yaml
- ingressroute-prometheus.yaml
- ingressroute-prometheus-http.yaml
- dashboard-agave-status.yaml
- prometheus-rules-agave.yaml

View File

@@ -0,0 +1,9 @@
apiVersion: traefik.io/v1alpha1
kind: Middleware
metadata:
name: redirect-to-https
namespace: monitoring
spec:
redirectScheme:
scheme: https
permanent: true

View File

@@ -0,0 +1,61 @@
apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
name: agave-rpc-alerts
namespace: monitoring
labels:
app: kube-prometheus-stack
release: monitoring-stack
spec:
groups:
- name: agave-rpc-health
rules:
- alert: AgaveRPCDown
expr: "max by (instance) (solana_rpc_up{job=\"mpabi-node-exporter\"}) == 0"
for: 30s
labels:
severity: critical
team: mpabi
annotations:
summary: "Agave RPC is unreachable"
description: "RPC probe from node exporter reports solana_rpc_up == 0 for instance {{ $labels.instance }}."
- alert: AgaveRPCSlotLagHigh
expr: "sum by (instance) (solana_rpc_slot_lag{job=\"mpabi-node-exporter\"}) > 50"
for: 2m
labels:
severity: warning
team: mpabi
annotations:
summary: "Agave RPC is lagging behind cluster"
description: "Current slot lag is {{ $value }} for instance {{ $labels.instance }}. Reference endpoint in probe config may be misconfigured or validator is behind."
- alert: AgaveRPCSlotLagCritical
expr: "sum by (instance) (solana_rpc_slot_lag{job=\"mpabi-node-exporter\"}) > 500"
for: 2m
labels:
severity: critical
team: mpabi
annotations:
summary: "Agave RPC severe lag"
description: "Slot lag is critically high ({{ $value }} slots) on instance {{ $labels.instance }}."
- alert: AgaveIOHigh
expr: |
sum by (instance) (
(rate(node_disk_read_bytes_total{job="mpabi-node-exporter",device=~"nvme.*"}[5m]) +
rate(node_disk_written_bytes_total{job="mpabi-node-exporter",device=~"nvme.*"}[5m])) / 1024 / 1024
) > 300
for: 5m
labels:
severity: warning
team: mpabi
annotations:
summary: "High storage I/O on Agave node"
description: "Combined NVMe read+write throughput >300 MiB/s for 5m on {{ $labels.instance }}. Check disk pressure and Geyser/ledger workload."
- alert: AgaveIOWaitHigh
expr: "avg by (instance) (rate(node_cpu_seconds_total{job=\"mpabi-node-exporter\",mode=\"iowait\"}[5m])) > 0.2"
for: 5m
labels:
severity: warning
team: mpabi
annotations:
summary: "High iowait on Agave node"
description: "Iowait over 20% on average for 5m on {{ $labels.instance }}. Storage latency is likely impacting slot progress."

View File

@@ -91,10 +91,10 @@ spec:
ingressClassName: traefik
tls:
- hosts:
- portainer.rv32i.pl
secretName: portainer-rv32i-pl-tls
- portainer.mpabi.pl
secretName: portainer-mpabi-pl-tls
rules:
- host: portainer.rv32i.pl
- host: portainer.mpabi.pl
http:
paths:
- path: /

View File

@@ -0,0 +1,18 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: trade-frontend
spec:
template:
spec:
containers:
- name: frontend
volumeMounts:
- name: frontend-server-script
mountPath: /app/services/frontend/server.mjs
subPath: frontend-server.mjs
readOnly: true
volumes:
- name: frontend-server-script
configMap:
name: trade-frontend-server-script

View File

@@ -0,0 +1,813 @@
import crypto from 'node:crypto';
import { spawnSync } from 'node:child_process';
import fs from 'node:fs';
import http from 'node:http';
import https from 'node:https';
import net from 'node:net';
import path from 'node:path';
import tls from 'node:tls';
const PORT = Number.parseInt(process.env.PORT || '8081', 10);
if (!Number.isInteger(PORT) || PORT <= 0) throw new Error(`Invalid PORT: ${process.env.PORT}`);
const APP_VERSION = String(process.env.APP_VERSION || 'v1').trim() || 'v1';
const BUILD_TIMESTAMP = String(process.env.BUILD_TIMESTAMP || '').trim() || undefined;
const STARTED_AT = new Date().toISOString();
const STATIC_DIR = process.env.STATIC_DIR || '/srv';
const BASIC_AUTH_FILE = process.env.BASIC_AUTH_FILE || '/tokens/frontend.json';
const API_READ_TOKEN_FILE = process.env.API_READ_TOKEN_FILE || '/tokens/read.json';
const API_UPSTREAM = process.env.API_UPSTREAM || process.env.API_URL || 'http://api:8787';
const HASURA_UPSTREAM = process.env.HASURA_UPSTREAM || 'http://hasura:8080';
const HASURA_GRAPHQL_PATH = process.env.HASURA_GRAPHQL_PATH || '/v1/graphql';
const GRAPHQL_CORS_ORIGIN = process.env.GRAPHQL_CORS_ORIGIN || process.env.CORS_ORIGIN || '*';
const BASIC_AUTH_MODE = String(process.env.BASIC_AUTH_MODE || 'on')
.trim()
.toLowerCase();
const BASIC_AUTH_ENABLED = !['off', 'false', '0', 'disabled', 'none'].includes(BASIC_AUTH_MODE);
const AUTH_USER_HEADER = String(process.env.AUTH_USER_HEADER || 'x-trade-user')
.trim()
.toLowerCase();
const AUTH_MODE = String(process.env.AUTH_MODE || 'session')
.trim()
.toLowerCase();
const HTPASSWD_FILE = String(process.env.HTPASSWD_FILE || '/auth/users').trim();
const AUTH_SESSION_SECRET_FILE = String(process.env.AUTH_SESSION_SECRET_FILE || '').trim() || null;
const AUTH_SESSION_COOKIE = String(process.env.AUTH_SESSION_COOKIE || 'trade_session')
.trim()
.toLowerCase();
const AUTH_SESSION_TTL_SECONDS = Number.parseInt(process.env.AUTH_SESSION_TTL_SECONDS || '43200', 10); // 12h
const DLOB_SOURCE_COOKIE = String(process.env.DLOB_SOURCE_COOKIE || 'trade_dlob_source').trim() || 'trade_dlob_source';
const DLOB_SOURCE_DEFAULT = String(process.env.DLOB_SOURCE_DEFAULT || 'mevnode').trim() || 'mevnode';
function readJson(filePath) {
const raw = fs.readFileSync(filePath, 'utf8');
return JSON.parse(raw);
}
function readText(filePath) {
return fs.readFileSync(filePath, 'utf8');
}
function timingSafeEqualStr(a, b) {
const aa = Buffer.from(String(a), 'utf8');
const bb = Buffer.from(String(b), 'utf8');
if (aa.length !== bb.length) return false;
return crypto.timingSafeEqual(aa, bb);
}
function timingSafeEqualBuf(a, b) {
if (!(a instanceof Uint8Array) || !(b instanceof Uint8Array)) return false;
if (a.length !== b.length) return false;
return crypto.timingSafeEqual(Buffer.from(a), Buffer.from(b));
}
function loadBasicAuth() {
const j = readJson(BASIC_AUTH_FILE);
const username = (j?.username || '').toString();
const password = (j?.password || '').toString();
if (!username || !password) throw new Error(`Invalid BASIC_AUTH_FILE: ${BASIC_AUTH_FILE}`);
return { username, password };
}
function loadApiReadToken() {
const j = readJson(API_READ_TOKEN_FILE);
const token = (j?.token || '').toString();
if (!token) throw new Error(`Invalid API_READ_TOKEN_FILE: ${API_READ_TOKEN_FILE}`);
return token;
}
function send(res, status, headers, body) {
res.statusCode = status;
for (const [k, v] of Object.entries(headers || {})) res.setHeader(k, v);
if (body == null) return void res.end();
res.end(body);
}
function sendJson(res, status, body) {
send(res, status, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' }, JSON.stringify(body));
}
function basicAuthRequired(res) {
res.setHeader('www-authenticate', 'Basic realm="trade"');
send(res, 401, { 'content-type': 'text/plain; charset=utf-8' }, 'unauthorized');
}
function unauthorized(res) {
sendJson(res, 401, { ok: false, error: 'unauthorized' });
}
function isAuthorized(req, creds) {
const auth = req.headers.authorization || '';
const m = String(auth).match(/^Basic\s+(.+)$/i);
if (!m?.[1]) return false;
let decoded;
try {
decoded = Buffer.from(m[1], 'base64').toString('utf8');
} catch {
return false;
}
const idx = decoded.indexOf(':');
if (idx < 0) return false;
const u = decoded.slice(0, idx);
const p = decoded.slice(idx + 1);
return timingSafeEqualStr(u, creds.username) && timingSafeEqualStr(p, creds.password);
}
const MIME = {
'.html': 'text/html; charset=utf-8',
'.css': 'text/css; charset=utf-8',
'.js': 'application/javascript; charset=utf-8',
'.mjs': 'application/javascript; charset=utf-8',
'.json': 'application/json; charset=utf-8',
'.svg': 'image/svg+xml',
'.png': 'image/png',
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.gif': 'image/gif',
'.ico': 'image/x-icon',
'.txt': 'text/plain; charset=utf-8',
'.map': 'application/json; charset=utf-8',
};
function contentTypeFor(filePath) {
return MIME[path.extname(filePath).toLowerCase()] || 'application/octet-stream';
}
function safePathFromUrlPath(urlPath) {
const decoded = decodeURIComponent(urlPath);
const cleaned = decoded.replace(/\0/g, '');
// strip leading slash so join() doesn't ignore STATIC_DIR
const rel = cleaned.replace(/^\/+/, '');
const normalized = path.normalize(rel);
// prevent traversal
if (normalized.startsWith('..') || path.isAbsolute(normalized)) return null;
return normalized;
}
function injectIndexHtml(html, { dlobSource, redirectPath }) {
const src = normalizeDlobSource(dlobSource) || 'mevnode';
const redirect = safeRedirectPath(redirectPath);
const hrefBase = `/prefs/dlob-source?redirect=${encodeURIComponent(redirect)}&set=`;
const styleActive = 'font-weight:700;text-decoration:underline;';
const styleInactive = 'font-weight:400;text-decoration:none;';
const snippet = `
<!-- trade: dlob source switch -->
<div style="position:fixed;right:12px;bottom:12px;z-index:2147483647;background:rgba(0,0,0,0.72);color:#fff;padding:8px 10px;border-radius:10px;font:12px/1.2 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;backdrop-filter:blur(6px);">
<span style="opacity:0.85;margin-right:6px;">DLOB</span>
<a href="${hrefBase}mevnode" style="color:#fff;${src === 'mevnode' ? styleActive : styleInactive}">mevnode</a>
<span style="opacity:0.6;margin:0 6px;">|</span>
<a href="${hrefBase}drift" style="color:#fff;${src === 'drift' ? styleActive : styleInactive}">drift</a>
</div>
`;
const bodyClose = /<\/body>/i;
if (bodyClose.test(html)) return html.replace(bodyClose, `${snippet}</body>`);
return `${html}\n${snippet}\n`;
}
function serveStatic(req, res) {
if (req.method !== 'GET' && req.method !== 'HEAD') {
send(res, 405, { 'content-type': 'text/plain; charset=utf-8' }, 'method_not_allowed');
return;
}
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
const rel = safePathFromUrlPath(url.pathname);
if (rel == null) {
send(res, 400, { 'content-type': 'text/plain; charset=utf-8' }, 'bad_request');
return;
}
const root = path.resolve(STATIC_DIR);
const fileCandidate = path.resolve(root, rel);
if (!fileCandidate.startsWith(root)) {
send(res, 400, { 'content-type': 'text/plain; charset=utf-8' }, 'bad_request');
return;
}
const trySend = (filePath) => {
try {
const st = fs.statSync(filePath);
if (st.isDirectory()) return trySend(path.join(filePath, 'index.html'));
res.statusCode = 200;
res.setHeader('content-type', contentTypeFor(filePath));
res.setHeader('cache-control', filePath.endsWith('index.html') ? 'no-cache' : 'public, max-age=31536000');
if (req.method === 'HEAD') return void res.end();
if (filePath.endsWith('index.html')) {
const html = fs.readFileSync(filePath, 'utf8');
const injected = injectIndexHtml(html, {
dlobSource: resolveDlobSource(req),
redirectPath: url.pathname + url.search,
});
res.end(injected);
return true;
}
fs.createReadStream(filePath).pipe(res);
return true;
} catch {
return false;
}
};
// exact file, otherwise SPA fallback
if (trySend(fileCandidate)) return;
const indexPath = path.join(root, 'index.html');
if (trySend(indexPath)) return;
send(res, 404, { 'content-type': 'text/plain; charset=utf-8' }, 'not_found');
}
function stripHopByHopHeaders(headers) {
const hop = new Set([
'connection',
'keep-alive',
'proxy-authenticate',
'proxy-authorization',
'te',
'trailer',
'transfer-encoding',
'upgrade',
]);
const out = {};
for (const [k, v] of Object.entries(headers || {})) {
if (hop.has(k.toLowerCase())) continue;
out[k] = v;
}
return out;
}
function readHeader(req, name) {
const v = req.headers[String(name).toLowerCase()];
return Array.isArray(v) ? v[0] : v;
}
function readCookie(req, name) {
const raw = typeof req.headers.cookie === 'string' ? req.headers.cookie : '';
if (!raw) return null;
const needle = `${name}=`;
for (const part of raw.split(';')) {
const t = part.trim();
if (!t.startsWith(needle)) continue;
return t.slice(needle.length) || null;
}
return null;
}
function normalizeDlobSource(value) {
const v = String(value ?? '')
.trim()
.toLowerCase();
if (v === 'mevnode') return 'mevnode';
if (v === 'drift') return 'drift';
return null;
}
function resolveDlobSource(req) {
const fromCookie = normalizeDlobSource(readCookie(req, DLOB_SOURCE_COOKIE));
if (fromCookie) return fromCookie;
return normalizeDlobSource(DLOB_SOURCE_DEFAULT) || 'mevnode';
}
function safeRedirectPath(value) {
const s = String(value ?? '').trim();
if (!s.startsWith('/')) return '/';
if (s.startsWith('//')) return '/';
return s.replace(/\r|\n/g, '');
}
function setDlobSourceCookie(res, { secure, dlobSource }) {
const src = normalizeDlobSource(dlobSource);
if (!src) return false;
const parts = [
`${DLOB_SOURCE_COOKIE}=${src}`,
'Path=/',
'SameSite=Lax',
'HttpOnly',
'Max-Age=31536000',
];
if (secure) parts.push('Secure');
res.setHeader('set-cookie', parts.join('; '));
return true;
}
function resolveAuthUser(req) {
const user = readHeader(req, AUTH_USER_HEADER) || readHeader(req, 'x-webauth-user');
const value = typeof user === 'string' ? user.trim() : '';
return value || null;
}
function isHttpsRequest(req) {
const xf = readHeader(req, 'x-forwarded-proto');
if (typeof xf === 'string' && xf.toLowerCase() === 'https') return true;
return Boolean(req.socket && req.socket.encrypted);
}
function base64urlEncode(buf) {
return Buffer.from(buf)
.toString('base64')
.replace(/\+/g, '-')
.replace(/\//g, '_')
.replace(/=+$/g, '');
}
function base64urlDecode(str) {
const cleaned = String(str).replace(/-/g, '+').replace(/_/g, '/');
const pad = cleaned.length % 4 === 0 ? '' : '='.repeat(4 - (cleaned.length % 4));
return Buffer.from(cleaned + pad, 'base64');
}
function loadSessionSecret() {
if (process.env.AUTH_SESSION_SECRET && String(process.env.AUTH_SESSION_SECRET).trim()) {
return Buffer.from(String(process.env.AUTH_SESSION_SECRET).trim(), 'utf8');
}
if (AUTH_SESSION_SECRET_FILE) {
try {
const txt = readText(AUTH_SESSION_SECRET_FILE).trim();
if (txt) return Buffer.from(txt, 'utf8');
} catch {
// ignore
}
}
return crypto.randomBytes(32);
}
const SESSION_SECRET = loadSessionSecret();
function signSessionPayload(payloadB64) {
return crypto.createHmac('sha256', SESSION_SECRET).update(payloadB64).digest();
}
function makeSessionCookieValue(username) {
const now = Math.floor(Date.now() / 1000);
const exp = now + (Number.isFinite(AUTH_SESSION_TTL_SECONDS) && AUTH_SESSION_TTL_SECONDS > 0 ? AUTH_SESSION_TTL_SECONDS : 43200);
const payload = JSON.stringify({ u: String(username), exp });
const payloadB64 = base64urlEncode(Buffer.from(payload, 'utf8'));
const sigB64 = base64urlEncode(signSessionPayload(payloadB64));
return `${payloadB64}.${sigB64}`;
}
function getSessionUser(req) {
const raw = readCookie(req, AUTH_SESSION_COOKIE);
if (!raw) return null;
const parts = raw.split('.');
if (parts.length !== 2) return null;
const [payloadB64, sigB64] = parts;
if (!payloadB64 || !sigB64) return null;
let payload;
try {
payload = JSON.parse(base64urlDecode(payloadB64).toString('utf8'));
} catch {
return null;
}
const u = typeof payload?.u === 'string' ? payload.u.trim() : '';
const exp = Number(payload?.exp);
if (!u || !Number.isFinite(exp)) return null;
const now = Math.floor(Date.now() / 1000);
if (now >= exp) return null;
const expected = signSessionPayload(payloadB64);
let got;
try {
got = base64urlDecode(sigB64);
} catch {
return null;
}
if (!timingSafeEqualBuf(expected, got)) return null;
return u;
}
function resolveAuthenticatedUser(req) {
const sessionUser = getSessionUser(req);
if (sessionUser) return sessionUser;
const headerUser = resolveAuthUser(req);
if (headerUser) return headerUser;
if (AUTH_MODE === 'off' || AUTH_MODE === 'none' || AUTH_MODE === 'disabled') return 'anonymous';
return null;
}
function clearSessionCookie(res, secure) {
const parts = [`${AUTH_SESSION_COOKIE}=`, 'Path=/', 'Max-Age=0', 'HttpOnly', 'SameSite=Lax'];
if (secure) parts.push('Secure');
res.setHeader('set-cookie', parts.join('; '));
}
function setSessionCookie(res, secure, username) {
const value = makeSessionCookieValue(username);
const parts = [
`${AUTH_SESSION_COOKIE}=${value}`,
'Path=/',
`Max-Age=${Number.isFinite(AUTH_SESSION_TTL_SECONDS) ? AUTH_SESSION_TTL_SECONDS : 43200}`,
'HttpOnly',
'SameSite=Lax',
];
if (secure) parts.push('Secure');
res.setHeader('set-cookie', parts.join('; '));
}
function verifyWithHtpasswd(username, password) {
try {
const r = spawnSync('htpasswd', ['-vb', HTPASSWD_FILE, String(username), String(password)], {
stdio: 'ignore',
timeout: 3000,
});
return r.status === 0;
} catch {
return false;
}
}
function readBody(req, limitBytes = 1024 * 16) {
return new Promise((resolve, reject) => {
let total = 0;
const chunks = [];
req.on('data', (chunk) => {
total += chunk.length;
if (total > limitBytes) {
reject(new Error('payload_too_large'));
req.destroy();
return;
}
chunks.push(chunk);
});
req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
req.on('error', reject);
});
}
function proxyApi(req, res, apiReadToken) {
const upstreamBase = new URL(API_UPSTREAM);
const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
const prefix = '/api';
const strippedPath = inUrl.pathname === prefix ? '/' : inUrl.pathname.startsWith(prefix + '/') ? inUrl.pathname.slice(prefix.length) : null;
if (strippedPath == null) {
send(res, 404, { 'content-type': 'text/plain; charset=utf-8' }, 'not_found');
return;
}
const target = new URL(upstreamBase.toString());
target.pathname = strippedPath || '/';
target.search = inUrl.search;
const isHttps = target.protocol === 'https:';
const lib = isHttps ? https : http;
const headers = stripHopByHopHeaders(req.headers);
delete headers.authorization; // basic auth from client must not leak upstream
headers.host = target.host;
headers.authorization = `Bearer ${apiReadToken}`;
const upstreamReq = lib.request(
{
protocol: target.protocol,
hostname: target.hostname,
port: target.port || (isHttps ? 443 : 80),
method: req.method,
path: target.pathname + target.search,
headers,
},
(upstreamRes) => {
const outHeaders = stripHopByHopHeaders(upstreamRes.headers);
res.writeHead(upstreamRes.statusCode || 502, outHeaders);
upstreamRes.pipe(res);
}
);
upstreamReq.on('error', (err) => {
if (!res.headersSent) {
send(res, 502, { 'content-type': 'text/plain; charset=utf-8' }, `bad_gateway: ${err?.message || err}`);
} else {
res.destroy();
}
});
req.pipe(upstreamReq);
}
function withCors(res) {
res.setHeader('access-control-allow-origin', GRAPHQL_CORS_ORIGIN);
res.setHeader('access-control-allow-methods', 'GET,POST,OPTIONS');
res.setHeader(
'access-control-allow-headers',
'content-type, authorization, x-hasura-admin-secret, x-hasura-role, x-hasura-user-id, x-hasura-dlob-source'
);
}
function proxyGraphqlHttp(req, res) {
const upstreamBase = new URL(HASURA_UPSTREAM);
const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
const target = new URL(upstreamBase.toString());
target.pathname = HASURA_GRAPHQL_PATH;
target.search = inUrl.search;
const isHttps = target.protocol === 'https:';
const lib = isHttps ? https : http;
const headers = stripHopByHopHeaders(req.headers);
headers.host = target.host;
delete headers['x-hasura-dlob-source'];
headers['x-hasura-dlob-source'] = resolveDlobSource(req);
const upstreamReq = lib.request(
{
protocol: target.protocol,
hostname: target.hostname,
port: target.port || (isHttps ? 443 : 80),
method: req.method,
path: target.pathname + target.search,
headers,
},
(upstreamRes) => {
const outHeaders = stripHopByHopHeaders(upstreamRes.headers);
withCors(res);
res.writeHead(upstreamRes.statusCode || 502, outHeaders);
upstreamRes.pipe(res);
}
);
upstreamReq.on('error', (err) => {
if (!res.headersSent) {
withCors(res);
send(res, 502, { 'content-type': 'text/plain; charset=utf-8' }, `bad_gateway: ${err?.message || err}`);
} else {
res.destroy();
}
});
req.pipe(upstreamReq);
}
function isGraphqlPath(pathname) {
return pathname === '/graphql' || pathname === '/graphql-ws';
}
function proxyGraphqlWs(req, socket, head) {
const upstreamBase = new URL(HASURA_UPSTREAM);
const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
const target = new URL(upstreamBase.toString());
target.pathname = HASURA_GRAPHQL_PATH;
target.search = inUrl.search;
const port = Number(target.port || (target.protocol === 'https:' ? 443 : 80));
const host = target.hostname;
const connect =
target.protocol === 'https:'
? () => tls.connect({ host, port, servername: host })
: () => net.connect({ host, port });
const upstream = connect();
upstream.setNoDelay(true);
socket.setNoDelay(true);
// For WebSocket upgrades we must forward `connection`/`upgrade` and related headers.
const headers = { ...req.headers };
delete headers['content-length'];
delete headers['content-type'];
headers.host = target.host;
delete headers['x-hasura-dlob-source'];
headers['x-hasura-dlob-source'] = resolveDlobSource(req);
const lines = [];
lines.push(`GET ${target.pathname + target.search} HTTP/1.1`);
for (const [k, v] of Object.entries(headers)) {
if (v == null) continue;
if (Array.isArray(v)) {
for (const vv of v) lines.push(`${k}: ${vv}`);
} else {
lines.push(`${k}: ${v}`);
}
}
lines.push('', '');
upstream.write(lines.join('\r\n'));
if (head?.length) upstream.write(head);
upstream.on('error', () => {
try {
socket.destroy();
} catch {
// ignore
}
});
socket.on('error', () => {
try {
upstream.destroy();
} catch {
// ignore
}
});
upstream.pipe(socket);
socket.pipe(upstream);
}
async function handler(req, res) {
if (req.method === 'GET' && (req.url === '/healthz' || req.url?.startsWith('/healthz?'))) {
send(
res,
200,
{ 'content-type': 'application/json; charset=utf-8' },
JSON.stringify({ ok: true, version: APP_VERSION, buildTimestamp: BUILD_TIMESTAMP, startedAt: STARTED_AT })
);
return;
}
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
if (isGraphqlPath(url.pathname)) {
if (req.method === 'OPTIONS') {
withCors(res);
res.statusCode = 204;
res.end();
return;
}
if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') {
const user = resolveAuthenticatedUser(req);
if (!user) {
withCors(res);
unauthorized(res);
return;
}
}
withCors(res);
proxyGraphqlHttp(req, res);
return;
}
if (req.method === 'GET' && url.pathname === '/whoami') {
sendJson(res, 200, { ok: true, user: resolveAuthenticatedUser(req), mode: AUTH_MODE });
return;
}
if (req.method === 'GET' && url.pathname === '/prefs/dlob-source') {
const set = url.searchParams.get('set');
if (!set) {
sendJson(res, 200, { ok: true, dlobSource: resolveDlobSource(req) });
return;
}
const ok = setDlobSourceCookie(res, { secure: isHttpsRequest(req), dlobSource: set });
if (!ok) {
sendJson(res, 400, { ok: false, error: 'invalid_dlob_source' });
return;
}
res.statusCode = 302;
res.setHeader('location', safeRedirectPath(url.searchParams.get('redirect') || '/'));
res.end();
return;
}
if (req.method === 'POST' && url.pathname === '/auth/login') {
if (AUTH_MODE === 'off' || AUTH_MODE === 'none' || AUTH_MODE === 'disabled') {
sendJson(res, 400, { ok: false, error: 'auth_disabled' });
return;
}
const raw = await readBody(req);
const ct = String(req.headers['content-type'] || '').toLowerCase();
let username = '';
let password = '';
if (ct.includes('application/json')) {
let json;
try {
json = JSON.parse(raw);
} catch {
sendJson(res, 400, { ok: false, error: 'bad_json' });
return;
}
username = typeof json?.username === 'string' ? json.username.trim() : '';
password = typeof json?.password === 'string' ? json.password : '';
} else {
const params = new URLSearchParams(raw);
username = String(params.get('username') || '').trim();
password = String(params.get('password') || '');
}
if (!username || !password || username.length > 64 || password.length > 200) {
sendJson(res, 400, { ok: false, error: 'invalid_input' });
return;
}
const ok = verifyWithHtpasswd(username, password);
if (!ok) {
unauthorized(res);
return;
}
const secure = isHttpsRequest(req);
setSessionCookie(res, secure, username);
sendJson(res, 200, { ok: true, user: username });
return;
}
if ((req.method === 'POST' || req.method === 'GET') && (url.pathname === '/auth/logout' || url.pathname === '/logout')) {
clearSessionCookie(res, isHttpsRequest(req));
if (req.method === 'GET') {
res.statusCode = 302;
res.setHeader('location', '/');
res.end();
return;
}
sendJson(res, 200, { ok: true });
return;
}
if (BASIC_AUTH_ENABLED) {
let creds;
try {
creds = loadBasicAuth();
} catch (e) {
send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e));
return;
}
if (!isAuthorized(req, creds)) {
basicAuthRequired(res);
return;
}
}
if (req.url?.startsWith('/api') && (req.url === '/api' || req.url.startsWith('/api/'))) {
if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') {
const user = resolveAuthenticatedUser(req);
if (!user) {
unauthorized(res);
return;
}
}
let token;
try {
token = loadApiReadToken();
} catch (e) {
send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e));
return;
}
proxyApi(req, res, token);
return;
}
serveStatic(req, res);
}
const server = http.createServer((req, res) => {
handler(req, res).catch((e) => {
if (res.headersSent) {
res.destroy();
return;
}
send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e));
});
});
server.on('upgrade', (req, socket, head) => {
try {
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
if (!isGraphqlPath(url.pathname)) {
socket.destroy();
return;
}
if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') {
const user = resolveAuthenticatedUser(req);
if (!user) {
try {
socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
} catch {
// ignore
}
socket.destroy();
return;
}
}
proxyGraphqlWs(req, socket, head);
} catch {
socket.destroy();
}
});
server.listen(PORT, () => {
console.log(
JSON.stringify(
{
service: 'trade-frontend',
port: PORT,
staticDir: STATIC_DIR,
apiUpstream: API_UPSTREAM,
hasuraUpstream: HASURA_UPSTREAM,
basicAuthFile: BASIC_AUTH_FILE,
basicAuthMode: BASIC_AUTH_MODE,
apiReadTokenFile: API_READ_TOKEN_FILE,
authUserHeader: AUTH_USER_HEADER,
authMode: AUTH_MODE,
htpasswdFile: HTPASSWD_FILE,
},
null,
2
)
);
});

View File

@@ -6,5 +6,16 @@ namespace: trade-prod
resources:
- ../../base
patchesStrategicMerge:
- frontend-graphql-proxy-patch.yaml
configMapGenerator:
- name: trade-frontend-server-script
files:
- frontend-server.mjs
generatorOptions:
disableNameSuffixHash: true
commonLabels:
env: prod

View File

@@ -0,0 +1,219 @@
import process from 'node:process';
import { setTimeout as sleep } from 'node:timers/promises';
function getIsoNow() {
return new Date().toISOString();
}
function envString(name, fallback) {
const v = process.env[name];
if (v == null) return fallback;
const s = String(v).trim();
return s ? s : fallback;
}
function envInt(name, fallback, { min, max } = {}) {
const v = process.env[name];
if (v == null) return fallback;
const n = Number.parseInt(String(v), 10);
if (!Number.isFinite(n)) return fallback;
const low = typeof min === 'number' ? min : n;
const high = typeof max === 'number' ? max : n;
return Math.max(low, Math.min(high, n));
}
function envList(name, fallbackCsv) {
const raw = process.env[name] ?? fallbackCsv;
return String(raw)
.split(',')
.map((s) => s.trim())
.filter(Boolean);
}
function toIntOrNull(v) {
if (v == null) return null;
if (typeof v === 'number') return Number.isFinite(v) ? Math.trunc(v) : null;
if (typeof v === 'string') {
const s = v.trim();
if (!s) return null;
const n = Number.parseInt(s, 10);
return Number.isFinite(n) ? n : null;
}
return null;
}
function numStr(v) {
if (v == null) return null;
if (typeof v === 'number') return Number.isFinite(v) ? String(v) : null;
if (typeof v === 'string') {
const s = v.trim();
return s ? s : null;
}
return null;
}
function isoFromEpochMs(v) {
const n = typeof v === 'number' ? v : typeof v === 'string' ? Number(v.trim()) : NaN;
if (!Number.isFinite(n) || n <= 0) return null;
const d = new Date(n);
const ms = d.getTime();
if (!Number.isFinite(ms)) return null;
return d.toISOString();
}
function resolveConfig() {
const hasuraUrl = envString('HASURA_GRAPHQL_URL', 'http://hasura:8080/v1/graphql');
const hasuraAdminSecret = envString('HASURA_ADMIN_SECRET', '');
if (!hasuraAdminSecret) throw new Error('Missing HASURA_ADMIN_SECRET');
const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
const pollMs = envInt('TICKS_POLL_MS', 1000, { min: 250, max: 60_000 });
const source = envString('TICKS_SOURCE', 'dlob_stats');
return { hasuraUrl, hasuraAdminSecret, markets, pollMs, source };
}
async function graphqlRequest(cfg, query, variables) {
const res = await fetch(cfg.hasuraUrl, {
method: 'POST',
headers: {
'content-type': 'application/json',
'x-hasura-admin-secret': cfg.hasuraAdminSecret,
},
body: JSON.stringify({ query, variables }),
signal: AbortSignal.timeout(10_000),
});
const text = await res.text();
if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`);
let json;
try {
json = JSON.parse(text);
} catch {
throw new Error(`Hasura: invalid json: ${text}`);
}
if (json.errors?.length) throw new Error(json.errors.map((e) => e.message).join(' | '));
return json.data;
}
async function fetchStats(cfg) {
const query = `
query DlobStatsLatest($markets: [String!]!) {
dlob_stats_latest(where: { market_name: { _in: $markets } }) {
market_name
market_index
ts
slot
oracle_price
mark_price
mid_price
best_bid_price
best_ask_price
updated_at
}
}
`;
const data = await graphqlRequest(cfg, query, { markets: cfg.markets });
const rows = Array.isArray(data?.dlob_stats_latest) ? data.dlob_stats_latest : [];
return rows;
}
async function insertTicks(cfg, objects) {
if (!objects.length) return 0;
const mutation = `
mutation InsertTicks($objects: [drift_ticks_insert_input!]!) {
insert_drift_ticks(objects: $objects) { affected_rows }
}
`;
const data = await graphqlRequest(cfg, mutation, { objects });
return Number(data?.insert_drift_ticks?.affected_rows || 0);
}
async function main() {
const cfg = resolveConfig();
const lastUpdatedAtByMarket = new Map();
console.log(
JSON.stringify(
{
service: 'trade-ingestor',
mode: 'dlob_stats_ticks',
startedAt: getIsoNow(),
hasuraUrl: cfg.hasuraUrl,
markets: cfg.markets,
pollMs: cfg.pollMs,
source: cfg.source,
},
null,
2
)
);
while (true) {
try {
const rows = await fetchStats(cfg);
const nowIso = getIsoNow();
const objects = [];
for (const r of rows) {
const marketName = String(r?.market_name || '').trim();
if (!marketName) continue;
const updatedAt = r?.updated_at ? String(r.updated_at) : '';
if (updatedAt && lastUpdatedAtByMarket.get(marketName) === updatedAt) continue;
if (updatedAt) lastUpdatedAtByMarket.set(marketName, updatedAt);
const marketIndex = toIntOrNull(r?.market_index) ?? 0;
const dlobIso = isoFromEpochMs(r?.ts);
const tsIso = dlobIso || nowIso;
const oraclePrice = numStr(r?.oracle_price) || numStr(r?.mark_price) || numStr(r?.mid_price);
const markPrice = numStr(r?.mark_price) || numStr(r?.mid_price) || oraclePrice;
if (!oraclePrice) continue;
objects.push({
ts: tsIso,
market_index: marketIndex,
symbol: marketName,
oracle_price: oraclePrice,
mark_price: markPrice,
oracle_slot: r?.slot == null ? null : String(r.slot),
source: cfg.source,
raw: {
from: 'dlob_stats_latest',
market_name: marketName,
market_index: marketIndex,
dlob: {
ts: r?.ts ?? null,
slot: r?.slot ?? null,
best_bid_price: r?.best_bid_price ?? null,
best_ask_price: r?.best_ask_price ?? null,
mid_price: r?.mid_price ?? null,
updated_at: updatedAt || null,
},
},
});
}
const inserted = await insertTicks(cfg, objects);
if (inserted) {
console.log(`[dlob-ticks] inserted=${inserted} ts=${nowIso}`);
}
} catch (err) {
console.error(`[dlob-ticks] error: ${String(err?.message || err)}`);
await sleep(2_000);
}
await sleep(cfg.pollMs);
}
}
main().catch((err) => {
console.error(String(err?.stack || err));
process.exitCode = 1;
});

View File

@@ -0,0 +1,487 @@
import fs from 'node:fs';
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function envString(name, fallback) {
const v = process.env[name];
if (v == null) return fallback;
const s = String(v).trim();
return s ? s : fallback;
}
function envNumber(name, fallback) {
const v = process.env[name];
if (v == null) return fallback;
const n = Number(v);
return Number.isFinite(n) ? n : fallback;
}
function envInt(name, fallback, { min, max } = {}) {
const v = process.env[name];
if (v == null) return fallback;
const n = Number.parseInt(String(v), 10);
if (!Number.isInteger(n)) return fallback;
const nn = Math.max(min ?? n, Math.min(max ?? n, n));
return nn;
}
function readJson(filePath) {
try {
const raw = fs.readFileSync(filePath, 'utf8');
return JSON.parse(raw);
} catch {
return undefined;
}
}
function readTokenFromFile(filePath) {
const json = readJson(filePath);
const raw = json?.token || json?.jwt || json?.authToken;
const tok = typeof raw === 'string' ? raw.trim() : '';
return tok ? tok : undefined;
}
function urlWithPath(baseUrl, path) {
const u = new URL(baseUrl);
const basePath = u.pathname && u.pathname !== '/' ? u.pathname.replace(/\/$/, '') : '';
u.pathname = `${basePath}${path}`;
return u;
}
async function httpJson(url, options) {
const res = await fetch(url, options);
const text = await res.text();
let json;
try {
json = JSON.parse(text);
} catch {
json = undefined;
}
return { ok: res.ok, status: res.status, json, text };
}
function mean(values) {
if (!values.length) return 0;
return values.reduce((a, b) => a + b, 0) / values.length;
}
function stddev(values) {
if (!values.length) return 0;
const m = mean(values);
const v = values.reduce((acc, x) => acc + (x - m) * (x - m), 0) / values.length;
return Math.sqrt(v);
}
function quantile(values, q) {
if (!values.length) return 0;
const sorted = values.slice().sort((a, b) => a - b);
const pos = (sorted.length - 1) * q;
const base = Math.floor(pos);
const rest = pos - base;
if (sorted[base + 1] == null) return sorted[base];
return sorted[base] + rest * (sorted[base + 1] - sorted[base]);
}
function randn() {
// BoxMuller
let u = 0;
let v = 0;
while (u === 0) u = Math.random();
while (v === 0) v = Math.random();
return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v);
}
function clamp(v, min, max) {
if (v < min) return min;
if (v > max) return max;
return v;
}
function isTruthy(value) {
const v = String(value ?? '')
.trim()
.toLowerCase();
if (!v) return false;
return !['0', 'false', 'off', 'no', 'none', 'disabled'].includes(v);
}
function parsePriceFromTick(t) {
const mp = t?.mark_price ?? t?.markPrice;
const op = t?.oracle_price ?? t?.oraclePrice ?? t?.price;
const v = mp != null ? Number(mp) : Number(op);
return Number.isFinite(v) && v > 0 ? v : null;
}
function computeLogReturns(prices) {
const out = [];
for (let i = 1; i < prices.length; i++) {
const a = prices[i - 1];
const b = prices[i];
if (!(a > 0) || !(b > 0)) continue;
const r = Math.log(b / a);
if (Number.isFinite(r)) out.push(r);
}
return out;
}
class BlockSampler {
#values;
#minBlock;
#maxBlock;
#idx = 0;
#end = 0;
#remaining = 0;
constructor(values, { minBlock, maxBlock }) {
this.#values = values.slice();
this.#minBlock = Math.max(1, minBlock);
this.#maxBlock = Math.max(this.#minBlock, maxBlock);
this.#startNewBlock();
}
#startNewBlock() {
const n = this.#values.length;
if (n <= 1) {
this.#idx = 0;
this.#end = n;
this.#remaining = n;
return;
}
const start = Math.floor(Math.random() * n);
const span = this.#maxBlock - this.#minBlock + 1;
const len = this.#minBlock + Math.floor(Math.random() * span);
this.#idx = start;
this.#end = Math.min(n, start + len);
this.#remaining = this.#end - this.#idx;
}
next() {
if (this.#values.length === 0) return 0;
if (this.#remaining <= 0 || this.#idx >= this.#end) this.#startNewBlock();
const v = this.#values[this.#idx];
this.#idx += 1;
this.#remaining -= 1;
return Number.isFinite(v) ? v : 0;
}
}
async function fetchSeedTicksPage({ apiBase, readToken, symbol, source, limit, to }) {
const u = urlWithPath(apiBase, '/v1/ticks');
u.searchParams.set('symbol', symbol);
u.searchParams.set('limit', String(limit));
if (source) u.searchParams.set('source', source);
if (to) u.searchParams.set('to', to);
const res = await httpJson(u.toString(), {
method: 'GET',
headers: {
authorization: `Bearer ${readToken}`,
},
});
if (!res.ok) throw new Error(`seed_ticks_http_${res.status}: ${res.text}`);
if (!res.json?.ok) throw new Error(`seed_ticks_error: ${res.json?.error || res.text}`);
const ticks = Array.isArray(res.json?.ticks) ? res.json.ticks : [];
return ticks;
}
async function fetchSeedTicksPaged({ apiBase, readToken, symbol, source, desiredLimit, pageLimit, maxPages }) {
const pages = [];
let cursorTo = null;
for (let page = 0; page < maxPages; page++) {
const remaining = desiredLimit - pages.reduce((acc, p) => acc + p.length, 0);
if (remaining <= 0) break;
const limit = Math.min(pageLimit, remaining);
const ticks = await fetchSeedTicksPage({ apiBase, readToken, symbol, source, limit, to: cursorTo });
if (!ticks.length) break;
// Server returns ascending ticks; we unshift to keep overall chronological order.
pages.unshift(ticks);
const oldestTs = String(ticks[0]?.ts || '').trim();
if (!oldestTs) break;
const oldestMs = Date.parse(oldestTs);
if (!Number.isFinite(oldestMs)) break;
cursorTo = new Date(oldestMs - 1).toISOString();
}
const flat = pages.flat();
if (!flat.length) return flat;
// Best-effort de-duplication (in case of overlapping `to` bounds).
const seen = new Set();
const out = [];
for (const t of flat) {
const ts = String(t?.ts || '');
const key = ts ? ts : JSON.stringify(t);
if (seen.has(key)) continue;
seen.add(key);
out.push(t);
}
return out;
}
async function ingestTick({ apiBase, writeToken, tick }) {
const u = urlWithPath(apiBase, '/v1/ingest/tick');
const res = await httpJson(u.toString(), {
method: 'POST',
headers: {
'content-type': 'application/json',
authorization: `Bearer ${writeToken}`,
},
body: JSON.stringify(tick),
});
if (!res.ok) throw new Error(`ingest_http_${res.status}: ${res.text}`);
if (!res.json?.ok) throw new Error(`ingest_error: ${res.json?.error || res.text}`);
return res.json?.id || null;
}
function formatNumeric(value) {
if (!Number.isFinite(value)) return '0';
// Avoid scientific notation for small values while keeping reasonable precision.
const abs = Math.abs(value);
if (abs === 0) return '0';
if (abs >= 1) return value.toFixed(8).replace(/\.?0+$/, '');
if (abs >= 0.01) return value.toFixed(10).replace(/\.?0+$/, '');
if (abs >= 0.0001) return value.toFixed(12).replace(/\.?0+$/, '');
return value.toFixed(16).replace(/\.?0+$/, '');
}
async function main() {
const apiBase = envString('INGEST_API_URL', 'http://trade-api:8787');
const symbol = envString('MARKET_NAME', envString('SYMBOL', 'PUMP-PERP'));
const source = envString('SOURCE', 'drift_oracle');
const intervalMs = envInt('INTERVAL_MS', 1000, { min: 50, max: 60_000 });
const readTokenFile = envString('FAKE_READ_TOKEN_FILE', envString('READ_TOKEN_FILE', '/tokens/read.json'));
const writeTokenFile = envString('FAKE_WRITE_TOKEN_FILE', envString('WRITE_TOKEN_FILE', '/app/tokens/alg.json'));
const seedLimitDesired = envInt('FAKE_SEED_LIMIT', 50_000, { min: 50, max: 200_000 });
const seedPageLimit = envInt('FAKE_SEED_PAGE_LIMIT', 5000, { min: 50, max: 5000 });
const seedMaxPages = envInt(
'FAKE_SEED_MAX_PAGES',
Math.ceil(seedLimitDesired / seedPageLimit) + 2,
{ min: 1, max: 200 }
);
const seedSourceRaw = process.env.FAKE_SEED_SOURCE;
const seedSource = seedSourceRaw == null ? source : String(seedSourceRaw).trim();
const minBlock = envInt('FAKE_BLOCK_MIN', 120, { min: 1, max: 50_000 });
const maxBlock = envInt('FAKE_BLOCK_MAX', 1200, { min: 1, max: 50_000 });
const volScale = envNumber('FAKE_VOL_SCALE', 1);
const noiseScale = envNumber('FAKE_NOISE_SCALE', 0.05);
const meanReversion = envNumber('FAKE_MEAN_REVERSION', 0.0002);
const markNoiseBps = envNumber('FAKE_MARK_NOISE_BPS', 5);
const logEvery = envInt('FAKE_LOG_EVERY', 30, { min: 1, max: 10_000 });
const marketIndexEnv = process.env.MARKET_INDEX;
const marketIndexFallback = Number.isInteger(Number(marketIndexEnv)) ? Number(marketIndexEnv) : 0;
const startPriceEnv = envNumber('FAKE_START_PRICE', 0);
const clampEnabled = isTruthy(process.env.FAKE_CLAMP ?? '1');
const clampQLow = clamp(envNumber('FAKE_CLAMP_Q_LOW', 0.05), 0.0, 0.49);
const clampQHigh = clamp(envNumber('FAKE_CLAMP_Q_HIGH', 0.95), 0.51, 1.0);
const clampLowMult = envNumber('FAKE_CLAMP_LOW_MULT', 0.8);
const clampHighMult = envNumber('FAKE_CLAMP_HIGH_MULT', 1.2);
const readToken = readTokenFromFile(readTokenFile);
const writeToken = readTokenFromFile(writeTokenFile);
if (!writeToken) throw new Error(`Missing write token (expected JSON token at ${writeTokenFile})`);
let seedTicks = [];
let seedPrices = [];
let marketIndex = marketIndexFallback;
let seedFromTs = null;
let seedToTs = null;
if (!readToken) {
console.warn(`[fake-ingestor] No read token at ${readTokenFile}; running without seed data.`);
} else {
seedTicks = await fetchSeedTicksPaged({
apiBase,
readToken,
symbol,
source: seedSource ? seedSource : undefined,
desiredLimit: seedLimitDesired,
pageLimit: seedPageLimit,
maxPages: seedMaxPages,
});
seedFromTs = seedTicks.length ? String(seedTicks[0]?.ts || '') : null;
seedToTs = seedTicks.length ? String(seedTicks[seedTicks.length - 1]?.ts || '') : null;
for (const t of seedTicks) {
const p = parsePriceFromTick(t);
if (p != null) seedPrices.push(p);
}
const lastTick = seedTicks.length ? seedTicks[seedTicks.length - 1] : null;
const mi = lastTick?.market_index ?? lastTick?.marketIndex;
if (Number.isInteger(Number(mi))) marketIndex = Number(mi);
}
if (!seedPrices.length && startPriceEnv > 0) {
seedPrices = [startPriceEnv];
}
if (!seedPrices.length) {
throw new Error(
'No seed prices available. Provide FAKE_START_PRICE (and optionally MARKET_INDEX) or mount a read token to seed from /v1/ticks.'
);
}
const returnsRaw = computeLogReturns(seedPrices);
const mu = mean(returnsRaw);
const sigma = stddev(returnsRaw);
const center = envString('FAKE_CENTER_RETURNS', '1') !== '0';
const returns = returnsRaw.length
? center
? returnsRaw.map((r) => r - mu)
: returnsRaw
: [];
const sampler = new BlockSampler(returns.length ? returns : [0], { minBlock, maxBlock });
const pLow = quantile(seedPrices, clampQLow);
const p50 = quantile(seedPrices, 0.5);
const pHigh = quantile(seedPrices, clampQHigh);
const clampMin = pLow > 0 ? pLow * clampLowMult : Math.min(...seedPrices) * clampLowMult;
const clampMax = pHigh > 0 ? pHigh * clampHighMult : Math.max(...seedPrices) * clampHighMult;
let logPrice = Math.log(seedPrices[seedPrices.length - 1]);
const targetLog = Math.log(p50 > 0 ? p50 : seedPrices[seedPrices.length - 1]);
console.log(
JSON.stringify(
{
service: 'trade-fake-ingestor',
apiBase,
symbol,
source,
intervalMs,
marketIndex,
seed: {
ok: Boolean(seedTicks.length),
desiredLimit: seedLimitDesired,
pageLimit: seedPageLimit,
maxPages: seedMaxPages,
source: seedSource || null,
ticks: seedTicks.length,
prices: seedPrices.length,
fromTs: seedFromTs,
toTs: seedToTs,
},
model: {
type: 'block-bootstrap-returns',
centerReturns: center,
mu,
sigma,
volScale,
noiseScale,
meanReversion,
block: { minBlock, maxBlock },
clamp: clampEnabled ? { min: clampMin, max: clampMax } : { disabled: true },
},
},
null,
2
)
);
let stopping = false;
process.on('SIGINT', () => {
stopping = true;
});
process.on('SIGTERM', () => {
stopping = true;
});
let tickCount = 0;
let nextAt = Date.now();
while (!stopping) {
const now = Date.now();
if (now < nextAt) await sleep(nextAt - now);
nextAt += intervalMs;
const r = sampler.next();
const noise = (Number.isFinite(sigma) ? sigma : 0) * noiseScale * randn();
const revert = Number.isFinite(meanReversion) ? meanReversion * (targetLog - logPrice) : 0;
const step = (Number.isFinite(r) ? r : 0) * volScale + noise + revert;
logPrice += step;
let oraclePrice = Math.exp(logPrice);
if (clampEnabled && Number.isFinite(clampMin) && Number.isFinite(clampMax) && clampMax > clampMin) {
oraclePrice = clamp(oraclePrice, clampMin, clampMax);
logPrice = Math.log(oraclePrice);
}
const markNoise = (markNoiseBps / 10_000) * randn();
const markPrice = oraclePrice * (1 + markNoise);
const ts = new Date().toISOString();
const tick = {
ts,
market_index: marketIndex,
symbol,
oracle_price: formatNumeric(oraclePrice),
mark_price: formatNumeric(markPrice),
source,
raw: {
fake: true,
model: 'block-bootstrap-returns',
seeded: seedTicks.length
? {
symbol,
source: seedSource || null,
desiredLimit: seedLimitDesired,
pageLimit: seedPageLimit,
maxPages: seedMaxPages,
fromTs: seedFromTs,
toTs: seedToTs,
}
: {
symbol,
source: null,
desiredLimit: 0,
pageLimit: seedPageLimit,
maxPages: seedMaxPages,
fromTs: null,
toTs: null,
},
},
};
try {
await ingestTick({ apiBase, writeToken, tick });
tickCount += 1;
if (tickCount % logEvery === 0) {
console.log(
`[fake-ingestor] ok ticks=${tickCount} ts=${ts} px=${tick.oracle_price} mark=${tick.mark_price} clamp=[${formatNumeric(
clampMin
)},${formatNumeric(clampMax)}]`
);
}
} catch (err) {
console.warn(`[fake-ingestor] ingest failed: ${String(err?.message || err)}`);
await sleep(Math.min(5000, Math.max(250, intervalMs)));
}
}
console.log('[fake-ingestor] stopped');
}
main().catch((err) => {
console.error(String(err?.stack || err));
process.exitCode = 1;
});

View File

@@ -0,0 +1,18 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: trade-frontend
spec:
template:
spec:
containers:
- name: frontend
volumeMounts:
- name: frontend-server-script
mountPath: /app/services/frontend/server.mjs
subPath: frontend-server.mjs
readOnly: true
volumes:
- name: frontend-server-script
configMap:
name: trade-frontend-server-script

View File

@@ -0,0 +1,24 @@
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: trade-frontend-root
annotations:
cert-manager.io/cluster-issuer: letsencrypt-prod
traefik.ingress.kubernetes.io/router.entrypoints: websecure
spec:
ingressClassName: traefik
tls:
- hosts:
- mpabi.pl
secretName: mpabi-pl-tls
rules:
- host: mpabi.pl
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: trade-frontend
port:
number: 8081

View File

@@ -9,10 +9,10 @@ spec:
ingressClassName: traefik
tls:
- hosts:
- trade.rv32i.pl
secretName: trade-rv32i-pl-tls
- trade.mpabi.pl
secretName: trade-mpabi-pl-tls
rules:
- host: trade.rv32i.pl
- host: trade.mpabi.pl
http:
paths:
- path: /

View File

@@ -0,0 +1,813 @@
import crypto from 'node:crypto';
import { spawnSync } from 'node:child_process';
import fs from 'node:fs';
import http from 'node:http';
import https from 'node:https';
import net from 'node:net';
import path from 'node:path';
import tls from 'node:tls';
const PORT = Number.parseInt(process.env.PORT || '8081', 10);
if (!Number.isInteger(PORT) || PORT <= 0) throw new Error(`Invalid PORT: ${process.env.PORT}`);
const APP_VERSION = String(process.env.APP_VERSION || 'v1').trim() || 'v1';
const BUILD_TIMESTAMP = String(process.env.BUILD_TIMESTAMP || '').trim() || undefined;
const STARTED_AT = new Date().toISOString();
const STATIC_DIR = process.env.STATIC_DIR || '/srv';
const BASIC_AUTH_FILE = process.env.BASIC_AUTH_FILE || '/tokens/frontend.json';
const API_READ_TOKEN_FILE = process.env.API_READ_TOKEN_FILE || '/tokens/read.json';
const API_UPSTREAM = process.env.API_UPSTREAM || process.env.API_URL || 'http://api:8787';
const HASURA_UPSTREAM = process.env.HASURA_UPSTREAM || 'http://hasura:8080';
const HASURA_GRAPHQL_PATH = process.env.HASURA_GRAPHQL_PATH || '/v1/graphql';
const GRAPHQL_CORS_ORIGIN = process.env.GRAPHQL_CORS_ORIGIN || process.env.CORS_ORIGIN || '*';
const BASIC_AUTH_MODE = String(process.env.BASIC_AUTH_MODE || 'on')
.trim()
.toLowerCase();
const BASIC_AUTH_ENABLED = !['off', 'false', '0', 'disabled', 'none'].includes(BASIC_AUTH_MODE);
const AUTH_USER_HEADER = String(process.env.AUTH_USER_HEADER || 'x-trade-user')
.trim()
.toLowerCase();
const AUTH_MODE = String(process.env.AUTH_MODE || 'session')
.trim()
.toLowerCase();
const HTPASSWD_FILE = String(process.env.HTPASSWD_FILE || '/auth/users').trim();
const AUTH_SESSION_SECRET_FILE = String(process.env.AUTH_SESSION_SECRET_FILE || '').trim() || null;
const AUTH_SESSION_COOKIE = String(process.env.AUTH_SESSION_COOKIE || 'trade_session')
.trim()
.toLowerCase();
const AUTH_SESSION_TTL_SECONDS = Number.parseInt(process.env.AUTH_SESSION_TTL_SECONDS || '43200', 10); // 12h
const DLOB_SOURCE_COOKIE = String(process.env.DLOB_SOURCE_COOKIE || 'trade_dlob_source').trim() || 'trade_dlob_source';
const DLOB_SOURCE_DEFAULT = String(process.env.DLOB_SOURCE_DEFAULT || 'mevnode').trim() || 'mevnode';
function readJson(filePath) {
const raw = fs.readFileSync(filePath, 'utf8');
return JSON.parse(raw);
}
function readText(filePath) {
return fs.readFileSync(filePath, 'utf8');
}
function timingSafeEqualStr(a, b) {
const aa = Buffer.from(String(a), 'utf8');
const bb = Buffer.from(String(b), 'utf8');
if (aa.length !== bb.length) return false;
return crypto.timingSafeEqual(aa, bb);
}
function timingSafeEqualBuf(a, b) {
if (!(a instanceof Uint8Array) || !(b instanceof Uint8Array)) return false;
if (a.length !== b.length) return false;
return crypto.timingSafeEqual(Buffer.from(a), Buffer.from(b));
}
function loadBasicAuth() {
const j = readJson(BASIC_AUTH_FILE);
const username = (j?.username || '').toString();
const password = (j?.password || '').toString();
if (!username || !password) throw new Error(`Invalid BASIC_AUTH_FILE: ${BASIC_AUTH_FILE}`);
return { username, password };
}
function loadApiReadToken() {
const j = readJson(API_READ_TOKEN_FILE);
const token = (j?.token || '').toString();
if (!token) throw new Error(`Invalid API_READ_TOKEN_FILE: ${API_READ_TOKEN_FILE}`);
return token;
}
function send(res, status, headers, body) {
res.statusCode = status;
for (const [k, v] of Object.entries(headers || {})) res.setHeader(k, v);
if (body == null) return void res.end();
res.end(body);
}
function sendJson(res, status, body) {
send(res, status, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' }, JSON.stringify(body));
}
function basicAuthRequired(res) {
res.setHeader('www-authenticate', 'Basic realm="trade"');
send(res, 401, { 'content-type': 'text/plain; charset=utf-8' }, 'unauthorized');
}
function unauthorized(res) {
sendJson(res, 401, { ok: false, error: 'unauthorized' });
}
function isAuthorized(req, creds) {
const auth = req.headers.authorization || '';
const m = String(auth).match(/^Basic\s+(.+)$/i);
if (!m?.[1]) return false;
let decoded;
try {
decoded = Buffer.from(m[1], 'base64').toString('utf8');
} catch {
return false;
}
const idx = decoded.indexOf(':');
if (idx < 0) return false;
const u = decoded.slice(0, idx);
const p = decoded.slice(idx + 1);
return timingSafeEqualStr(u, creds.username) && timingSafeEqualStr(p, creds.password);
}
const MIME = {
'.html': 'text/html; charset=utf-8',
'.css': 'text/css; charset=utf-8',
'.js': 'application/javascript; charset=utf-8',
'.mjs': 'application/javascript; charset=utf-8',
'.json': 'application/json; charset=utf-8',
'.svg': 'image/svg+xml',
'.png': 'image/png',
'.jpg': 'image/jpeg',
'.jpeg': 'image/jpeg',
'.gif': 'image/gif',
'.ico': 'image/x-icon',
'.txt': 'text/plain; charset=utf-8',
'.map': 'application/json; charset=utf-8',
};
function contentTypeFor(filePath) {
return MIME[path.extname(filePath).toLowerCase()] || 'application/octet-stream';
}
function safePathFromUrlPath(urlPath) {
const decoded = decodeURIComponent(urlPath);
const cleaned = decoded.replace(/\0/g, '');
// strip leading slash so join() doesn't ignore STATIC_DIR
const rel = cleaned.replace(/^\/+/, '');
const normalized = path.normalize(rel);
// prevent traversal
if (normalized.startsWith('..') || path.isAbsolute(normalized)) return null;
return normalized;
}
function injectIndexHtml(html, { dlobSource, redirectPath }) {
const src = normalizeDlobSource(dlobSource) || 'mevnode';
const redirect = safeRedirectPath(redirectPath);
const hrefBase = `/prefs/dlob-source?redirect=${encodeURIComponent(redirect)}&set=`;
const styleActive = 'font-weight:700;text-decoration:underline;';
const styleInactive = 'font-weight:400;text-decoration:none;';
const snippet = `
<!-- trade: dlob source switch -->
<div style="position:fixed;right:12px;bottom:12px;z-index:2147483647;background:rgba(0,0,0,0.72);color:#fff;padding:8px 10px;border-radius:10px;font:12px/1.2 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;backdrop-filter:blur(6px);">
<span style="opacity:0.85;margin-right:6px;">DLOB</span>
<a href="${hrefBase}mevnode" style="color:#fff;${src === 'mevnode' ? styleActive : styleInactive}">mevnode</a>
<span style="opacity:0.6;margin:0 6px;">|</span>
<a href="${hrefBase}drift" style="color:#fff;${src === 'drift' ? styleActive : styleInactive}">drift</a>
</div>
`;
const bodyClose = /<\/body>/i;
if (bodyClose.test(html)) return html.replace(bodyClose, `${snippet}</body>`);
return `${html}\n${snippet}\n`;
}
function serveStatic(req, res) {
if (req.method !== 'GET' && req.method !== 'HEAD') {
send(res, 405, { 'content-type': 'text/plain; charset=utf-8' }, 'method_not_allowed');
return;
}
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
const rel = safePathFromUrlPath(url.pathname);
if (rel == null) {
send(res, 400, { 'content-type': 'text/plain; charset=utf-8' }, 'bad_request');
return;
}
const root = path.resolve(STATIC_DIR);
const fileCandidate = path.resolve(root, rel);
if (!fileCandidate.startsWith(root)) {
send(res, 400, { 'content-type': 'text/plain; charset=utf-8' }, 'bad_request');
return;
}
const trySend = (filePath) => {
try {
const st = fs.statSync(filePath);
if (st.isDirectory()) return trySend(path.join(filePath, 'index.html'));
res.statusCode = 200;
res.setHeader('content-type', contentTypeFor(filePath));
res.setHeader('cache-control', filePath.endsWith('index.html') ? 'no-cache' : 'public, max-age=31536000');
if (req.method === 'HEAD') return void res.end();
if (filePath.endsWith('index.html')) {
const html = fs.readFileSync(filePath, 'utf8');
const injected = injectIndexHtml(html, {
dlobSource: resolveDlobSource(req),
redirectPath: url.pathname + url.search,
});
res.end(injected);
return true;
}
fs.createReadStream(filePath).pipe(res);
return true;
} catch {
return false;
}
};
// exact file, otherwise SPA fallback
if (trySend(fileCandidate)) return;
const indexPath = path.join(root, 'index.html');
if (trySend(indexPath)) return;
send(res, 404, { 'content-type': 'text/plain; charset=utf-8' }, 'not_found');
}
function stripHopByHopHeaders(headers) {
const hop = new Set([
'connection',
'keep-alive',
'proxy-authenticate',
'proxy-authorization',
'te',
'trailer',
'transfer-encoding',
'upgrade',
]);
const out = {};
for (const [k, v] of Object.entries(headers || {})) {
if (hop.has(k.toLowerCase())) continue;
out[k] = v;
}
return out;
}
function readHeader(req, name) {
const v = req.headers[String(name).toLowerCase()];
return Array.isArray(v) ? v[0] : v;
}
function readCookie(req, name) {
const raw = typeof req.headers.cookie === 'string' ? req.headers.cookie : '';
if (!raw) return null;
const needle = `${name}=`;
for (const part of raw.split(';')) {
const t = part.trim();
if (!t.startsWith(needle)) continue;
return t.slice(needle.length) || null;
}
return null;
}
function normalizeDlobSource(value) {
const v = String(value ?? '')
.trim()
.toLowerCase();
if (v === 'mevnode') return 'mevnode';
if (v === 'drift') return 'drift';
return null;
}
function resolveDlobSource(req) {
const fromCookie = normalizeDlobSource(readCookie(req, DLOB_SOURCE_COOKIE));
if (fromCookie) return fromCookie;
return normalizeDlobSource(DLOB_SOURCE_DEFAULT) || 'mevnode';
}
function safeRedirectPath(value) {
const s = String(value ?? '').trim();
if (!s.startsWith('/')) return '/';
if (s.startsWith('//')) return '/';
return s.replace(/\r|\n/g, '');
}
function setDlobSourceCookie(res, { secure, dlobSource }) {
const src = normalizeDlobSource(dlobSource);
if (!src) return false;
const parts = [
`${DLOB_SOURCE_COOKIE}=${src}`,
'Path=/',
'SameSite=Lax',
'HttpOnly',
'Max-Age=31536000',
];
if (secure) parts.push('Secure');
res.setHeader('set-cookie', parts.join('; '));
return true;
}
function resolveAuthUser(req) {
const user = readHeader(req, AUTH_USER_HEADER) || readHeader(req, 'x-webauth-user');
const value = typeof user === 'string' ? user.trim() : '';
return value || null;
}
function isHttpsRequest(req) {
const xf = readHeader(req, 'x-forwarded-proto');
if (typeof xf === 'string' && xf.toLowerCase() === 'https') return true;
return Boolean(req.socket && req.socket.encrypted);
}
function base64urlEncode(buf) {
return Buffer.from(buf)
.toString('base64')
.replace(/\+/g, '-')
.replace(/\//g, '_')
.replace(/=+$/g, '');
}
function base64urlDecode(str) {
const cleaned = String(str).replace(/-/g, '+').replace(/_/g, '/');
const pad = cleaned.length % 4 === 0 ? '' : '='.repeat(4 - (cleaned.length % 4));
return Buffer.from(cleaned + pad, 'base64');
}
function loadSessionSecret() {
if (process.env.AUTH_SESSION_SECRET && String(process.env.AUTH_SESSION_SECRET).trim()) {
return Buffer.from(String(process.env.AUTH_SESSION_SECRET).trim(), 'utf8');
}
if (AUTH_SESSION_SECRET_FILE) {
try {
const txt = readText(AUTH_SESSION_SECRET_FILE).trim();
if (txt) return Buffer.from(txt, 'utf8');
} catch {
// ignore
}
}
return crypto.randomBytes(32);
}
const SESSION_SECRET = loadSessionSecret();
function signSessionPayload(payloadB64) {
return crypto.createHmac('sha256', SESSION_SECRET).update(payloadB64).digest();
}
function makeSessionCookieValue(username) {
const now = Math.floor(Date.now() / 1000);
const exp = now + (Number.isFinite(AUTH_SESSION_TTL_SECONDS) && AUTH_SESSION_TTL_SECONDS > 0 ? AUTH_SESSION_TTL_SECONDS : 43200);
const payload = JSON.stringify({ u: String(username), exp });
const payloadB64 = base64urlEncode(Buffer.from(payload, 'utf8'));
const sigB64 = base64urlEncode(signSessionPayload(payloadB64));
return `${payloadB64}.${sigB64}`;
}
function getSessionUser(req) {
const raw = readCookie(req, AUTH_SESSION_COOKIE);
if (!raw) return null;
const parts = raw.split('.');
if (parts.length !== 2) return null;
const [payloadB64, sigB64] = parts;
if (!payloadB64 || !sigB64) return null;
let payload;
try {
payload = JSON.parse(base64urlDecode(payloadB64).toString('utf8'));
} catch {
return null;
}
const u = typeof payload?.u === 'string' ? payload.u.trim() : '';
const exp = Number(payload?.exp);
if (!u || !Number.isFinite(exp)) return null;
const now = Math.floor(Date.now() / 1000);
if (now >= exp) return null;
const expected = signSessionPayload(payloadB64);
let got;
try {
got = base64urlDecode(sigB64);
} catch {
return null;
}
if (!timingSafeEqualBuf(expected, got)) return null;
return u;
}
function resolveAuthenticatedUser(req) {
const sessionUser = getSessionUser(req);
if (sessionUser) return sessionUser;
const headerUser = resolveAuthUser(req);
if (headerUser) return headerUser;
if (AUTH_MODE === 'off' || AUTH_MODE === 'none' || AUTH_MODE === 'disabled') return 'anonymous';
return null;
}
function clearSessionCookie(res, secure) {
const parts = [`${AUTH_SESSION_COOKIE}=`, 'Path=/', 'Max-Age=0', 'HttpOnly', 'SameSite=Lax'];
if (secure) parts.push('Secure');
res.setHeader('set-cookie', parts.join('; '));
}
function setSessionCookie(res, secure, username) {
const value = makeSessionCookieValue(username);
const parts = [
`${AUTH_SESSION_COOKIE}=${value}`,
'Path=/',
`Max-Age=${Number.isFinite(AUTH_SESSION_TTL_SECONDS) ? AUTH_SESSION_TTL_SECONDS : 43200}`,
'HttpOnly',
'SameSite=Lax',
];
if (secure) parts.push('Secure');
res.setHeader('set-cookie', parts.join('; '));
}
function verifyWithHtpasswd(username, password) {
try {
const r = spawnSync('htpasswd', ['-vb', HTPASSWD_FILE, String(username), String(password)], {
stdio: 'ignore',
timeout: 3000,
});
return r.status === 0;
} catch {
return false;
}
}
function readBody(req, limitBytes = 1024 * 16) {
return new Promise((resolve, reject) => {
let total = 0;
const chunks = [];
req.on('data', (chunk) => {
total += chunk.length;
if (total > limitBytes) {
reject(new Error('payload_too_large'));
req.destroy();
return;
}
chunks.push(chunk);
});
req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
req.on('error', reject);
});
}
function proxyApi(req, res, apiReadToken) {
const upstreamBase = new URL(API_UPSTREAM);
const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
const prefix = '/api';
const strippedPath = inUrl.pathname === prefix ? '/' : inUrl.pathname.startsWith(prefix + '/') ? inUrl.pathname.slice(prefix.length) : null;
if (strippedPath == null) {
send(res, 404, { 'content-type': 'text/plain; charset=utf-8' }, 'not_found');
return;
}
const target = new URL(upstreamBase.toString());
target.pathname = strippedPath || '/';
target.search = inUrl.search;
const isHttps = target.protocol === 'https:';
const lib = isHttps ? https : http;
const headers = stripHopByHopHeaders(req.headers);
delete headers.authorization; // basic auth from client must not leak upstream
headers.host = target.host;
headers.authorization = `Bearer ${apiReadToken}`;
const upstreamReq = lib.request(
{
protocol: target.protocol,
hostname: target.hostname,
port: target.port || (isHttps ? 443 : 80),
method: req.method,
path: target.pathname + target.search,
headers,
},
(upstreamRes) => {
const outHeaders = stripHopByHopHeaders(upstreamRes.headers);
res.writeHead(upstreamRes.statusCode || 502, outHeaders);
upstreamRes.pipe(res);
}
);
upstreamReq.on('error', (err) => {
if (!res.headersSent) {
send(res, 502, { 'content-type': 'text/plain; charset=utf-8' }, `bad_gateway: ${err?.message || err}`);
} else {
res.destroy();
}
});
req.pipe(upstreamReq);
}
function withCors(res) {
res.setHeader('access-control-allow-origin', GRAPHQL_CORS_ORIGIN);
res.setHeader('access-control-allow-methods', 'GET,POST,OPTIONS');
res.setHeader(
'access-control-allow-headers',
'content-type, authorization, x-hasura-admin-secret, x-hasura-role, x-hasura-user-id, x-hasura-dlob-source'
);
}
function proxyGraphqlHttp(req, res) {
const upstreamBase = new URL(HASURA_UPSTREAM);
const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
const target = new URL(upstreamBase.toString());
target.pathname = HASURA_GRAPHQL_PATH;
target.search = inUrl.search;
const isHttps = target.protocol === 'https:';
const lib = isHttps ? https : http;
const headers = stripHopByHopHeaders(req.headers);
headers.host = target.host;
delete headers['x-hasura-dlob-source'];
headers['x-hasura-dlob-source'] = resolveDlobSource(req);
const upstreamReq = lib.request(
{
protocol: target.protocol,
hostname: target.hostname,
port: target.port || (isHttps ? 443 : 80),
method: req.method,
path: target.pathname + target.search,
headers,
},
(upstreamRes) => {
const outHeaders = stripHopByHopHeaders(upstreamRes.headers);
withCors(res);
res.writeHead(upstreamRes.statusCode || 502, outHeaders);
upstreamRes.pipe(res);
}
);
upstreamReq.on('error', (err) => {
if (!res.headersSent) {
withCors(res);
send(res, 502, { 'content-type': 'text/plain; charset=utf-8' }, `bad_gateway: ${err?.message || err}`);
} else {
res.destroy();
}
});
req.pipe(upstreamReq);
}
function isGraphqlPath(pathname) {
return pathname === '/graphql' || pathname === '/graphql-ws';
}
function proxyGraphqlWs(req, socket, head) {
const upstreamBase = new URL(HASURA_UPSTREAM);
const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
const target = new URL(upstreamBase.toString());
target.pathname = HASURA_GRAPHQL_PATH;
target.search = inUrl.search;
const port = Number(target.port || (target.protocol === 'https:' ? 443 : 80));
const host = target.hostname;
const connect =
target.protocol === 'https:'
? () => tls.connect({ host, port, servername: host })
: () => net.connect({ host, port });
const upstream = connect();
upstream.setNoDelay(true);
socket.setNoDelay(true);
// For WebSocket upgrades we must forward `connection`/`upgrade` and related headers.
const headers = { ...req.headers };
delete headers['content-length'];
delete headers['content-type'];
headers.host = target.host;
delete headers['x-hasura-dlob-source'];
headers['x-hasura-dlob-source'] = resolveDlobSource(req);
const lines = [];
lines.push(`GET ${target.pathname + target.search} HTTP/1.1`);
for (const [k, v] of Object.entries(headers)) {
if (v == null) continue;
if (Array.isArray(v)) {
for (const vv of v) lines.push(`${k}: ${vv}`);
} else {
lines.push(`${k}: ${v}`);
}
}
lines.push('', '');
upstream.write(lines.join('\r\n'));
if (head?.length) upstream.write(head);
upstream.on('error', () => {
try {
socket.destroy();
} catch {
// ignore
}
});
socket.on('error', () => {
try {
upstream.destroy();
} catch {
// ignore
}
});
upstream.pipe(socket);
socket.pipe(upstream);
}
async function handler(req, res) {
if (req.method === 'GET' && (req.url === '/healthz' || req.url?.startsWith('/healthz?'))) {
send(
res,
200,
{ 'content-type': 'application/json; charset=utf-8' },
JSON.stringify({ ok: true, version: APP_VERSION, buildTimestamp: BUILD_TIMESTAMP, startedAt: STARTED_AT })
);
return;
}
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
if (isGraphqlPath(url.pathname)) {
if (req.method === 'OPTIONS') {
withCors(res);
res.statusCode = 204;
res.end();
return;
}
if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') {
const user = resolveAuthenticatedUser(req);
if (!user) {
withCors(res);
unauthorized(res);
return;
}
}
withCors(res);
proxyGraphqlHttp(req, res);
return;
}
if (req.method === 'GET' && url.pathname === '/whoami') {
sendJson(res, 200, { ok: true, user: resolveAuthenticatedUser(req), mode: AUTH_MODE });
return;
}
if (req.method === 'GET' && url.pathname === '/prefs/dlob-source') {
const set = url.searchParams.get('set');
if (!set) {
sendJson(res, 200, { ok: true, dlobSource: resolveDlobSource(req) });
return;
}
const ok = setDlobSourceCookie(res, { secure: isHttpsRequest(req), dlobSource: set });
if (!ok) {
sendJson(res, 400, { ok: false, error: 'invalid_dlob_source' });
return;
}
res.statusCode = 302;
res.setHeader('location', safeRedirectPath(url.searchParams.get('redirect') || '/'));
res.end();
return;
}
if (req.method === 'POST' && url.pathname === '/auth/login') {
if (AUTH_MODE === 'off' || AUTH_MODE === 'none' || AUTH_MODE === 'disabled') {
sendJson(res, 400, { ok: false, error: 'auth_disabled' });
return;
}
const raw = await readBody(req);
const ct = String(req.headers['content-type'] || '').toLowerCase();
let username = '';
let password = '';
if (ct.includes('application/json')) {
let json;
try {
json = JSON.parse(raw);
} catch {
sendJson(res, 400, { ok: false, error: 'bad_json' });
return;
}
username = typeof json?.username === 'string' ? json.username.trim() : '';
password = typeof json?.password === 'string' ? json.password : '';
} else {
const params = new URLSearchParams(raw);
username = String(params.get('username') || '').trim();
password = String(params.get('password') || '');
}
if (!username || !password || username.length > 64 || password.length > 200) {
sendJson(res, 400, { ok: false, error: 'invalid_input' });
return;
}
const ok = verifyWithHtpasswd(username, password);
if (!ok) {
unauthorized(res);
return;
}
const secure = isHttpsRequest(req);
setSessionCookie(res, secure, username);
sendJson(res, 200, { ok: true, user: username });
return;
}
if ((req.method === 'POST' || req.method === 'GET') && (url.pathname === '/auth/logout' || url.pathname === '/logout')) {
clearSessionCookie(res, isHttpsRequest(req));
if (req.method === 'GET') {
res.statusCode = 302;
res.setHeader('location', '/');
res.end();
return;
}
sendJson(res, 200, { ok: true });
return;
}
if (BASIC_AUTH_ENABLED) {
let creds;
try {
creds = loadBasicAuth();
} catch (e) {
send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e));
return;
}
if (!isAuthorized(req, creds)) {
basicAuthRequired(res);
return;
}
}
if (req.url?.startsWith('/api') && (req.url === '/api' || req.url.startsWith('/api/'))) {
if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') {
const user = resolveAuthenticatedUser(req);
if (!user) {
unauthorized(res);
return;
}
}
let token;
try {
token = loadApiReadToken();
} catch (e) {
send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e));
return;
}
proxyApi(req, res, token);
return;
}
serveStatic(req, res);
}
const server = http.createServer((req, res) => {
handler(req, res).catch((e) => {
if (res.headersSent) {
res.destroy();
return;
}
send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e));
});
});
server.on('upgrade', (req, socket, head) => {
try {
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
if (!isGraphqlPath(url.pathname)) {
socket.destroy();
return;
}
if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') {
const user = resolveAuthenticatedUser(req);
if (!user) {
try {
socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
} catch {
// ignore
}
socket.destroy();
return;
}
}
proxyGraphqlWs(req, socket, head);
} catch {
socket.destroy();
}
});
server.listen(PORT, () => {
console.log(
JSON.stringify(
{
service: 'trade-frontend',
port: PORT,
staticDir: STATIC_DIR,
apiUpstream: API_UPSTREAM,
hasuraUpstream: HASURA_UPSTREAM,
basicAuthFile: BASIC_AUTH_FILE,
basicAuthMode: BASIC_AUTH_MODE,
apiReadTokenFile: API_READ_TOKEN_FILE,
authUserHeader: AUTH_USER_HEADER,
authMode: AUTH_MODE,
htpasswdFile: HTPASSWD_FILE,
},
null,
2
)
);
});

View File

@@ -12,3 +12,5 @@ spec:
value: "true"
- name: HASURA_GRAPHQL_DEV_MODE
value: "true"
- name: HASURA_GRAPHQL_CORS_DOMAIN
value: "http://localhost:5173,http://127.0.0.1:5173,https://trade.mpabi.pl"

View File

@@ -0,0 +1,36 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: trade-ingestor
spec:
template:
spec:
containers:
- name: ingestor
image: node:20-slim
imagePullPolicy: IfNotPresent
env:
- name: HASURA_GRAPHQL_URL
value: http://hasura:8080/v1/graphql
- name: HASURA_ADMIN_SECRET
valueFrom:
secretKeyRef:
name: trade-hasura
key: HASURA_GRAPHQL_ADMIN_SECRET
- name: DLOB_MARKETS
value: SOL-PERP,DOGE-PERP,JUP-PERP
- name: TICKS_POLL_MS
value: "1000"
- name: TICKS_SOURCE
value: "dlob_stats"
command: ["node"]
args: ["/opt/dlob/dlob-ingestor.mjs"]
volumeMounts:
- name: dlob-script
mountPath: /opt/dlob/dlob-ingestor.mjs
subPath: dlob-ingestor.mjs
readOnly: true
volumes:
- name: dlob-script
configMap:
name: trade-dlob-ingestor-script

View File

@@ -0,0 +1,54 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: trade-ingestor
spec:
template:
spec:
containers:
- name: ingestor
env:
- name: FAKE_READ_TOKEN_FILE
value: "/tokens/read.json"
- name: FAKE_WRITE_TOKEN_FILE
value: "/app/tokens/alg.json"
- name: FAKE_SEED_LIMIT
value: "50000"
- name: FAKE_SEED_PAGE_LIMIT
value: "5000"
- name: FAKE_SEED_MAX_PAGES
value: "20"
- name: FAKE_BLOCK_MIN
value: "300"
- name: FAKE_BLOCK_MAX
value: "1800"
- name: FAKE_VOL_SCALE
value: "1.8"
- name: FAKE_NOISE_SCALE
value: "0.03"
- name: FAKE_MEAN_REVERSION
value: "0.0001"
- name: FAKE_CLAMP_Q_LOW
value: "0.02"
- name: FAKE_CLAMP_Q_HIGH
value: "0.98"
- name: FAKE_CLAMP_LOW_MULT
value: "0.7"
- name: FAKE_CLAMP_HIGH_MULT
value: "1.3"
command: ["node"]
args: ["/opt/fake/fake-ingestor.mjs"]
volumeMounts:
- name: fake-script
mountPath: /opt/fake
readOnly: true
- name: read-tokens
mountPath: /tokens
readOnly: true
volumes:
- name: fake-script
configMap:
name: trade-fake-ingestor-script
- name: read-tokens
secret:
secretName: trade-frontend-tokens

View File

@@ -7,10 +7,24 @@ resources:
- ../../base
- pgadmin.yaml
- frontend-ingress.yaml
- frontend-ingress-root.yaml
patchesStrategicMerge:
- hasura-patch.yaml
- frontend-auth-patch.yaml
- frontend-graphql-proxy-patch.yaml
- ingestor-dlob-patch.yaml
configMapGenerator:
- name: trade-dlob-ingestor-script
files:
- dlob-ingestor.mjs
- name: trade-frontend-server-script
files:
- frontend-server.mjs
generatorOptions:
disableNameSuffixHash: true
commonLabels:
env: staging