From bb6f633d1982b8b2b41c209206a808250a50c620 Mon Sep 17 00:00:00 2001 From: mpabi Date: Fri, 13 Mar 2026 22:33:28 +0100 Subject: [PATCH] feat(kustomize): add mevnode bot overlay --- kustomize/overlays/mevnode-bot/README.md | 49 ++++ .../overlays/mevnode-bot/dlob-ingestor.mjs | 219 ++++++++++++++++++ .../overlays/mevnode-bot/kustomization.yaml | 66 ++++++ kustomize/overlays/mevnode-bot/namespace.yaml | 4 + kustomize/overlays/mevnode-bot/patch-api.yaml | 8 + .../mevnode-bot/patch-dlob-depth-worker.yaml | 12 + .../mevnode-bot/patch-dlob-publisher.yaml | 31 +++ .../mevnode-bot/patch-dlob-server.yaml | 8 + .../patch-dlob-slippage-worker.yaml | 12 + .../mevnode-bot/patch-dlob-worker.yaml | 12 + .../mevnode-bot/patch-frontend-service.yaml | 11 + .../overlays/mevnode-bot/patch-frontend.yaml | 8 + .../overlays/mevnode-bot/patch-ingestor.yaml | 39 ++++ 13 files changed, 479 insertions(+) create mode 100644 kustomize/overlays/mevnode-bot/README.md create mode 100644 kustomize/overlays/mevnode-bot/dlob-ingestor.mjs create mode 100644 kustomize/overlays/mevnode-bot/kustomization.yaml create mode 100644 kustomize/overlays/mevnode-bot/namespace.yaml create mode 100644 kustomize/overlays/mevnode-bot/patch-api.yaml create mode 100644 kustomize/overlays/mevnode-bot/patch-dlob-depth-worker.yaml create mode 100644 kustomize/overlays/mevnode-bot/patch-dlob-publisher.yaml create mode 100644 kustomize/overlays/mevnode-bot/patch-dlob-server.yaml create mode 100644 kustomize/overlays/mevnode-bot/patch-dlob-slippage-worker.yaml create mode 100644 kustomize/overlays/mevnode-bot/patch-dlob-worker.yaml create mode 100644 kustomize/overlays/mevnode-bot/patch-frontend-service.yaml create mode 100644 kustomize/overlays/mevnode-bot/patch-frontend.yaml create mode 100644 kustomize/overlays/mevnode-bot/patch-ingestor.yaml diff --git a/kustomize/overlays/mevnode-bot/README.md b/kustomize/overlays/mevnode-bot/README.md new file mode 100644 index 0000000..921f7e6 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/README.md @@ -0,0 +1,49 @@ +# mevnode-bot overlay + +To jest docelowy overlay dla runtime `mevnode_bot`. + +Zakres: + +- `postgres` +- `hasura` +- `trade-api` +- `trade-frontend` (`NodePort 30081`) +- `trade-ingestor` +- `dlob-redis` +- `dlob-publisher` +- `dlob-server` +- `dlob-worker` +- `dlob-depth-worker` +- `dlob-slippage-worker` + +Założenia runtime: + +- `mevnode_sol` wystawia po `wg0`: + - `RPC`: `http://10.91.0.1:8899` + - `WS`: `ws://10.91.0.1:8900` + - `Yellowstone`: `http://10.91.0.1:10000` +- `mevnode_bot` uruchamia tylko warstwę aplikacyjną. +- `trade-frontend` jest wystawiany przez `NodePort 30081`. +- Namespace pozostaje `trade-staging`. + +Wymagane sekrety: + +- `trade-postgres` +- `trade-hasura` +- `trade-api` +- `trade-frontend-tokens` +- `trade-dlob-rpc` + +Opcjonalne: + +- `gitea-registry` nie jest tu wymagane, bo overlay czyści `imagePullSecrets`. + +Uwagi: + +- Overlay zastępuje lokalny `_tmp/bot-rx-stack`. +- `trade-deploy/kustomize/overlays/mevnode-bot` jest teraz źródłem prawdy dla runtime `mevnode_bot`. +- Ten overlay nadal korzysta z plików z `../../base` w `configMapGenerator`, więc przy lokalnym `kustomize` może być potrzebne: + +```bash +sudo k3s kubectl kustomize trade-deploy/kustomize/overlays/mevnode-bot --load-restrictor LoadRestrictionsNone | sudo k3s kubectl apply -f - +``` diff --git a/kustomize/overlays/mevnode-bot/dlob-ingestor.mjs b/kustomize/overlays/mevnode-bot/dlob-ingestor.mjs new file mode 100644 index 0000000..8517237 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/dlob-ingestor.mjs @@ -0,0 +1,219 @@ +import process from 'node:process'; +import { setTimeout as sleep } from 'node:timers/promises'; + +function getIsoNow() { + return new Date().toISOString(); +} + +function envString(name, fallback) { + const v = process.env[name]; + if (v == null) return fallback; + const s = String(v).trim(); + return s ? s : fallback; +} + +function envInt(name, fallback, { min, max } = {}) { + const v = process.env[name]; + if (v == null) return fallback; + const n = Number.parseInt(String(v), 10); + if (!Number.isFinite(n)) return fallback; + const low = typeof min === 'number' ? min : n; + const high = typeof max === 'number' ? max : n; + return Math.max(low, Math.min(high, n)); +} + +function envList(name, fallbackCsv) { + const raw = process.env[name] ?? fallbackCsv; + return String(raw) + .split(',') + .map((s) => s.trim()) + .filter(Boolean); +} + +function toIntOrNull(v) { + if (v == null) return null; + if (typeof v === 'number') return Number.isFinite(v) ? Math.trunc(v) : null; + if (typeof v === 'string') { + const s = v.trim(); + if (!s) return null; + const n = Number.parseInt(s, 10); + return Number.isFinite(n) ? n : null; + } + return null; +} + +function numStr(v) { + if (v == null) return null; + if (typeof v === 'number') return Number.isFinite(v) ? String(v) : null; + if (typeof v === 'string') { + const s = v.trim(); + return s ? s : null; + } + return null; +} + +function isoFromEpochMs(v) { + const n = typeof v === 'number' ? v : typeof v === 'string' ? Number(v.trim()) : NaN; + if (!Number.isFinite(n) || n <= 0) return null; + const d = new Date(n); + const ms = d.getTime(); + if (!Number.isFinite(ms)) return null; + return d.toISOString(); +} + +function resolveConfig() { + const hasuraUrl = envString('HASURA_GRAPHQL_URL', 'http://hasura:8080/v1/graphql'); + const hasuraAdminSecret = envString('HASURA_ADMIN_SECRET', ''); + if (!hasuraAdminSecret) throw new Error('Missing HASURA_ADMIN_SECRET'); + + const markets = envList('DLOB_MARKETS', 'SOL-PERP,PUMP-PERP'); + const pollMs = envInt('TICKS_POLL_MS', 1000, { min: 250, max: 60000 }); + const source = envString('TICKS_SOURCE', 'dlob_stats'); + + return { hasuraUrl, hasuraAdminSecret, markets, pollMs, source }; +} + +async function graphqlRequest(cfg, query, variables) { + const res = await fetch(cfg.hasuraUrl, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-hasura-admin-secret': cfg.hasuraAdminSecret, + }, + body: JSON.stringify({ query, variables }), + signal: AbortSignal.timeout(10000), + }); + + const text = await res.text(); + if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`); + + let json; + try { + json = JSON.parse(text); + } catch { + throw new Error(`Hasura: invalid json: ${text}`); + } + + if (json.errors?.length) throw new Error(json.errors.map((e) => e.message).join(' | ')); + return json.data; +} + +async function fetchStats(cfg) { + const query = ` + query DlobStatsLatest($markets: [String!]!) { + dlob_stats_latest(where: { market_name: { _in: $markets } }) { + market_name + market_index + ts + slot + oracle_price + mark_price + mid_price + best_bid_price + best_ask_price + updated_at + } + } + `; + + const data = await graphqlRequest(cfg, query, { markets: cfg.markets }); + const rows = Array.isArray(data?.dlob_stats_latest) ? data.dlob_stats_latest : []; + return rows; +} + +async function insertTicks(cfg, objects) { + if (!objects.length) return 0; + + const mutation = ` + mutation InsertTicks($objects: [drift_ticks_insert_input!]!) { + insert_drift_ticks(objects: $objects) { affected_rows } + } + `; + + const data = await graphqlRequest(cfg, mutation, { objects }); + return Number(data?.insert_drift_ticks?.affected_rows || 0); +} + +async function main() { + const cfg = resolveConfig(); + const lastUpdatedAtByMarket = new Map(); + + console.log( + JSON.stringify( + { + service: 'trade-ingestor', + mode: 'dlob_stats_ticks', + startedAt: getIsoNow(), + hasuraUrl: cfg.hasuraUrl, + markets: cfg.markets, + pollMs: cfg.pollMs, + source: cfg.source, + }, + null, + 2 + ) + ); + + while (true) { + try { + const rows = await fetchStats(cfg); + const nowIso = getIsoNow(); + + const objects = []; + for (const r of rows) { + const marketName = String(r?.market_name || '').trim(); + if (!marketName) continue; + + const updatedAt = r?.updated_at ? String(r.updated_at) : ''; + if (updatedAt && lastUpdatedAtByMarket.get(marketName) === updatedAt) continue; + if (updatedAt) lastUpdatedAtByMarket.set(marketName, updatedAt); + + const marketIndex = toIntOrNull(r?.market_index) ?? 0; + const dlobIso = isoFromEpochMs(r?.ts); + const tsIso = dlobIso || nowIso; + + const oraclePrice = numStr(r?.oracle_price) || numStr(r?.mark_price) || numStr(r?.mid_price); + const markPrice = numStr(r?.mark_price) || numStr(r?.mid_price) || oraclePrice; + if (!oraclePrice) continue; + + objects.push({ + ts: tsIso, + market_index: marketIndex, + symbol: marketName, + oracle_price: oraclePrice, + mark_price: markPrice, + oracle_slot: r?.slot == null ? null : String(r.slot), + source: cfg.source, + raw: { + from: 'dlob_stats_latest', + market_name: marketName, + market_index: marketIndex, + dlob: { + ts: r?.ts ?? null, + slot: r?.slot ?? null, + best_bid_price: r?.best_bid_price ?? null, + best_ask_price: r?.best_ask_price ?? null, + mid_price: r?.mid_price ?? null, + updated_at: updatedAt || null, + }, + }, + }); + } + + const inserted = await insertTicks(cfg, objects); + if (inserted) { + console.log(`[dlob-ticks] inserted=${inserted} ts=${nowIso}`); + } + } catch (err) { + console.error(`[dlob-ticks] error: ${String(err?.message || err)}`); + await sleep(2000); + } + + await sleep(cfg.pollMs); + } +} + +main().catch((err) => { + console.error(String(err?.stack || err)); + process.exitCode = 1; +}); diff --git a/kustomize/overlays/mevnode-bot/kustomization.yaml b/kustomize/overlays/mevnode-bot/kustomization.yaml new file mode 100644 index 0000000..b4378c1 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/kustomization.yaml @@ -0,0 +1,66 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: trade-staging + +resources: + - namespace.yaml + - ../../base/cm-trade-deploy.yaml + - ../../base/postgres/service.yaml + - ../../base/postgres/statefulset.yaml + - ../../base/postgres/job-migrate.yaml + - ../../base/hasura/service.yaml + - ../../base/hasura/deployment.yaml + - ../../base/hasura/job-bootstrap.yaml + - ../../base/api/service.yaml + - ../../base/api/deployment.yaml + - ../../base/ingestor/deployment.yaml + - ../../base/frontend/service.yaml + - ../../base/frontend/deployment.yaml + - ../../base/dlob/redis.yaml + - ../../base/dlob/publisher-deployment.yaml + - ../../base/dlob/server-service.yaml + - ../../base/dlob/server-deployment.yaml + - ../../base/dlob-worker/deployment.yaml + - ../../base/dlob-depth-worker/deployment.yaml + - ../../base/dlob-slippage-worker/deployment.yaml + +patchesStrategicMerge: + - patch-api.yaml + - patch-frontend.yaml + - patch-frontend-service.yaml + - patch-ingestor.yaml + - patch-dlob-publisher.yaml + - patch-dlob-server.yaml + - patch-dlob-worker.yaml + - patch-dlob-depth-worker.yaml + - patch-dlob-slippage-worker.yaml + +configMapGenerator: + - name: postgres-initdb + files: + - ../../base/initdb/001_init.sql + - name: hasura-bootstrap-script + files: + - ../../base/hasura/hasura-bootstrap.mjs + - name: trade-api-wrapper + files: + - ../../base/api/wrapper.mjs + - name: trade-api-upstream + files: + - ../../base/api/server.mjs + - name: dlob-worker-script + files: + - ../../base/dlob-worker/worker.mjs + - name: dlob-depth-worker-script + files: + - ../../base/dlob-depth-worker/worker.mjs + - name: dlob-slippage-worker-script + files: + - ../../base/dlob-slippage-worker/worker.mjs + - name: trade-dlob-ingestor-script + files: + - dlob-ingestor.mjs + +generatorOptions: + disableNameSuffixHash: true diff --git a/kustomize/overlays/mevnode-bot/namespace.yaml b/kustomize/overlays/mevnode-bot/namespace.yaml new file mode 100644 index 0000000..886d807 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: trade-staging diff --git a/kustomize/overlays/mevnode-bot/patch-api.yaml b/kustomize/overlays/mevnode-bot/patch-api.yaml new file mode 100644 index 0000000..ced2120 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/patch-api.yaml @@ -0,0 +1,8 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trade-api +spec: + template: + spec: + imagePullSecrets: [] diff --git a/kustomize/overlays/mevnode-bot/patch-dlob-depth-worker.yaml b/kustomize/overlays/mevnode-bot/patch-dlob-depth-worker.yaml new file mode 100644 index 0000000..32db728 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/patch-dlob-depth-worker.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dlob-depth-worker +spec: + template: + spec: + containers: + - name: worker + env: + - name: DLOB_MARKETS + value: SOL-PERP,PUMP-PERP diff --git a/kustomize/overlays/mevnode-bot/patch-dlob-publisher.yaml b/kustomize/overlays/mevnode-bot/patch-dlob-publisher.yaml new file mode 100644 index 0000000..50a5974 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/patch-dlob-publisher.yaml @@ -0,0 +1,31 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dlob-publisher +spec: + template: + spec: + imagePullSecrets: [] + containers: + - name: publisher + env: + - name: PERP_MARKETS_TO_LOAD + value: "0,75" + - name: USE_GRPC + value: "true" + - name: USE_WEBSOCKET + value: "false" + - name: DISABLE_GPA_REFRESH + value: "true" + - name: GRPC_CLIENT + value: "yellowstone" + - name: GRPC_ENDPOINT + valueFrom: + secretKeyRef: + name: trade-dlob-rpc + key: GRPC_ENDPOINT + - name: TOKEN + valueFrom: + secretKeyRef: + name: trade-dlob-rpc + key: TOKEN diff --git a/kustomize/overlays/mevnode-bot/patch-dlob-server.yaml b/kustomize/overlays/mevnode-bot/patch-dlob-server.yaml new file mode 100644 index 0000000..1fae8d3 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/patch-dlob-server.yaml @@ -0,0 +1,8 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dlob-server +spec: + template: + spec: + imagePullSecrets: [] diff --git a/kustomize/overlays/mevnode-bot/patch-dlob-slippage-worker.yaml b/kustomize/overlays/mevnode-bot/patch-dlob-slippage-worker.yaml new file mode 100644 index 0000000..f982f70 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/patch-dlob-slippage-worker.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dlob-slippage-worker +spec: + template: + spec: + containers: + - name: worker + env: + - name: DLOB_MARKETS + value: SOL-PERP,PUMP-PERP diff --git a/kustomize/overlays/mevnode-bot/patch-dlob-worker.yaml b/kustomize/overlays/mevnode-bot/patch-dlob-worker.yaml new file mode 100644 index 0000000..a6d7781 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/patch-dlob-worker.yaml @@ -0,0 +1,12 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dlob-worker +spec: + template: + spec: + containers: + - name: worker + env: + - name: DLOB_MARKETS + value: SOL-PERP,PUMP-PERP diff --git a/kustomize/overlays/mevnode-bot/patch-frontend-service.yaml b/kustomize/overlays/mevnode-bot/patch-frontend-service.yaml new file mode 100644 index 0000000..4e65cdf --- /dev/null +++ b/kustomize/overlays/mevnode-bot/patch-frontend-service.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: trade-frontend +spec: + type: NodePort + ports: + - name: http + port: 8081 + targetPort: http + nodePort: 30081 diff --git a/kustomize/overlays/mevnode-bot/patch-frontend.yaml b/kustomize/overlays/mevnode-bot/patch-frontend.yaml new file mode 100644 index 0000000..2012ee5 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/patch-frontend.yaml @@ -0,0 +1,8 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trade-frontend +spec: + template: + spec: + imagePullSecrets: [] diff --git a/kustomize/overlays/mevnode-bot/patch-ingestor.yaml b/kustomize/overlays/mevnode-bot/patch-ingestor.yaml new file mode 100644 index 0000000..9854ac8 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/patch-ingestor.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: trade-ingestor +spec: + template: + spec: + imagePullSecrets: [] + containers: + - name: ingestor + image: node:20-slim + imagePullPolicy: IfNotPresent + env: + - name: HASURA_GRAPHQL_URL + value: http://hasura:8080/v1/graphql + - name: HASURA_ADMIN_SECRET + valueFrom: + secretKeyRef: + name: trade-hasura + key: HASURA_GRAPHQL_ADMIN_SECRET + - name: DLOB_MARKETS + value: SOL-PERP,PUMP-PERP + - name: TICKS_POLL_MS + value: "1000" + - name: TICKS_SOURCE + value: "dlob_stats" + command: + - node + args: + - /opt/dlob/dlob-ingestor.mjs + volumeMounts: + - name: dlob-script + mountPath: /opt/dlob/dlob-ingestor.mjs + subPath: dlob-ingestor.mjs + readOnly: true + volumes: + - name: dlob-script + configMap: + name: trade-dlob-ingestor-script