feat(hasura): expose canonical dlob projections

chore(dlob): enable publisher with canonical store
chore(dlob): enable websocket feed on bot overlay
2026-03-13 23:42:18 +01:00 · 2026-03-13 23:39:30 +01:00 · 2026-03-13 23:32:20 +01:00 · 2026-03-13 23:28:18 +01:00 · 2026-03-13 23:18:00 +01:00 · 2026-03-13 22:33:28 +01:00
25 changed files with 826 additions and 153 deletions
--- a/kustomize/base/api/server.mjs
+++ b/kustomize/base/api/server.mjs
@@ -744,7 +744,7 @@ function computeCandleFlowMovesFromTicks({ candle, bucketSeconds, points, rows,
 }

 function parseTimeframeToSeconds(tf) {
-  const v = normalizeChartTimeframe(tf);
+  const v = String(tf || '').trim().toLowerCase();
  if (!v) return 60;
  const m = v.match(/^(\d+)(s|m|h|d)$/);
  if (!m) throw new Error(`invalid_tf`);
@@ -755,40 +755,6 @@ function parseTimeframeToSeconds(tf) {
  return num * mult;
 }

-const SUPPORTED_CHART_TIMEFRAMES = new Set([
-  '1s',
-  '5s',
-  '15s',
-  '30s',
-  '1m',
-  '5m',
-  '15m',
-  '30m',
-  '1h',
-  '4h',
-  '8h',
-  '12h',
-  '1d',
-]);
-
-function normalizeChartTimeframe(tf) {
-  const raw = String(tf || '').trim();
-  if (!raw) return '1m';
-
-  const aliased =
-    raw === '1D' || raw === '24h' || raw === '24H'
-      ? '1d'
-      : raw === '1H'
-        ? '1h'
-        : raw.toLowerCase();
-
-  if (!SUPPORTED_CHART_TIMEFRAMES.has(aliased)) {
-    throw new Error('invalid_tf');
-  }
-
-  return aliased;
-}
-
 function fillForwardCandles(candles, { bucketSeconds, limit, nowSec }) {
  if (!Array.isArray(candles) || candles.length === 0) return [];
  if (!Number.isFinite(bucketSeconds) || bucketSeconds <= 0) return candles;
--- a/kustomize/base/dlob/publisher-deployment.yaml
+++ b/kustomize/base/dlob/publisher-deployment.yaml
@@ -18,7 +18,7 @@ spec:
        - name: gitea-registry
      containers:
        - name: publisher
-          image: gitea.mpabi.pl/trade/trade-dlob-server:sha-8a378b7-lite-l2
+          image: gitea.mpabi.pl/trade/trade-dlob-server:sha-5d55631-canonical
          imagePullPolicy: IfNotPresent
          ports:
            - name: http
--- a/kustomize/base/dlob/server-deployment.yaml
+++ b/kustomize/base/dlob/server-deployment.yaml
@@ -18,7 +18,7 @@ spec:
        - name: gitea-registry
      containers:
        - name: server
-          image: gitea.mpabi.pl/trade/trade-dlob-server:sha-8a378b7-lite-l2
+          image: gitea.mpabi.pl/trade/trade-dlob-server:sha-5d55631-canonical
          imagePullPolicy: IfNotPresent
          ports:
            - name: http
--- a/kustomize/base/hasura/hasura-bootstrap.mjs
+++ b/kustomize/base/hasura/hasura-bootstrap.mjs
@@ -92,14 +92,15 @@ async function main() {
  const PUBLIC_DLOB_SOURCE_HEADER = 'X-Hasura-Dlob-Source';

  const apiTokensTable = { schema: 'public', name: 'api_tokens' };
-  const botConfigTable = { schema: 'public', name: 'bot_config' };
-  const botStateTable = { schema: 'public', name: 'bot_state' };
-  const botEventsTable = { schema: 'public', name: 'bot_events' };
  const source = 'default';

  const baseTicks = { schema: 'public', name: 'drift_ticks' };
  const dlobL2LatestTable = { schema: 'public', name: 'dlob_l2_latest' };
+  const dlobL2LatestProjectionView = { schema: 'public', name: 'dlob_l2_latest_projection' };
+  const dlobL3LatestProjectionView = { schema: 'public', name: 'dlob_l3_latest_projection' };
+  const dlobBestMakersLatestProjectionView = { schema: 'public', name: 'dlob_best_makers_latest_projection' };
  const dlobStatsLatestTable = { schema: 'public', name: 'dlob_stats_latest' };
+  const dlobStatsLatestProjectionView = { schema: 'public', name: 'dlob_stats_latest_projection' };
  const dlobDepthBpsLatestTable = { schema: 'public', name: 'dlob_depth_bps_latest' };
  const dlobSlippageLatestTable = { schema: 'public', name: 'dlob_slippage_latest' };
  const dlobSlippageLatestV2Table = { schema: 'public', name: 'dlob_slippage_latest_v2' };
@@ -264,9 +265,36 @@ async function main() {
    await metadataIgnore({ type: 'pg_drop_update_permission', args: { source, table, role: 'ingestor' } });
  }

+  async function ensureProjectedPublicSelectTable(table, columns, { publicFilter, customName } = {}) {
+    await metadataIgnore({ type: 'pg_untrack_table', args: { source, table, cascade: true } });
+    const args = { source, table };
+    if (customName) {
+      args.configuration = { custom_name: customName };
+    }
+    await metadata({ type: 'pg_track_table', args });
+
+    await metadataIgnore({ type: 'pg_drop_select_permission', args: { source, table, role: 'public' } });
+    await metadata({
+      type: 'pg_create_select_permission',
+      args: {
+        source,
+        table,
+        role: 'public',
+        permission: {
+          columns,
+          filter: publicFilter || {},
+        },
+      },
+    });
+
+    await metadataIgnore({ type: 'pg_drop_insert_permission', args: { source, table, role: 'ingestor' } });
+    await metadataIgnore({ type: 'pg_drop_update_permission', args: { source, table, role: 'ingestor' } });
+  }
+
  const dlobPublicFilter = { source: { _eq: PUBLIC_DLOB_SOURCE_HEADER } };

-  await ensureDlobTable(dlobL2LatestTable, [
+  await metadataIgnore({ type: 'pg_untrack_table', args: { source, table: dlobL2LatestTable, cascade: true } });
+  await ensureProjectedPublicSelectTable(dlobL2LatestProjectionView, [
    'source',
    'market_name',
    'market_type',
@@ -281,9 +309,10 @@ async function main() {
    'asks',
    'raw',
    'updated_at',
-  ], { publicFilter: dlobPublicFilter });
+  ], { publicFilter: dlobPublicFilter, customName: 'dlob_l2_latest' });

-  await ensureDlobTable(dlobStatsLatestTable, [
+  await metadataIgnore({ type: 'pg_untrack_table', args: { source, table: dlobStatsLatestTable, cascade: true } });
+  await ensureProjectedPublicSelectTable(dlobStatsLatestProjectionView, [
    'source',
    'market_name',
    'market_type',
@@ -305,6 +334,31 @@ async function main() {
    'imbalance',
    'raw',
    'updated_at',
+  ], { publicFilter: dlobPublicFilter, customName: 'dlob_stats_latest' });
+
+  await ensureProjectedPublicSelectTable(dlobL3LatestProjectionView, [
+    'source',
+    'market_name',
+    'market_type',
+    'market_index',
+    'ts',
+    'slot',
+    'bids',
+    'asks',
+    'raw',
+    'updated_at',
+  ], { publicFilter: dlobPublicFilter });
+
+  await ensureProjectedPublicSelectTable(dlobBestMakersLatestProjectionView, [
+    'source',
+    'market_name',
+    'market_type',
+    'market_index',
+    'slot',
+    'bids',
+    'asks',
+    'raw',
+    'updated_at',
  ], { publicFilter: dlobPublicFilter });

  await ensurePublicSelectTable(dlobDepthBpsLatestTable, [
@@ -463,12 +517,6 @@ async function main() {
    'raw',
  ], { publicFilter: dlobPublicFilter });

-  // Bot control-plane tables (tracked; permissions are intentionally not created here by default).
-  for (const t of [botConfigTable, botStateTable, botEventsTable]) {
-    await metadataIgnore({ type: 'pg_untrack_table', args: { source, table: t } });
-    await metadata({ type: 'pg_track_table', args: { source, table: t } });
-  }
-
  // Return table type for candle functions (needed for Hasura to track the function).
  await metadataIgnore({ type: 'pg_track_table', args: { source, table: candlesReturnTable } });

--- a/kustomize/base/initdb/001_init.sql
+++ b/kustomize/base/initdb/001_init.sql
@@ -111,50 +111,6 @@ ALTER TABLE api_tokens ADD COLUMN IF NOT EXISTS scopes TEXT[] NOT NULL DEFAULT A
 CREATE INDEX IF NOT EXISTS api_tokens_revoked_at_idx
  ON api_tokens (revoked_at);

-- Bot control-plane (desired state) + executor telemetry (state/events).
--
-- MVP intent:
-- - `bot_config`: desired state (mode/market/limits/kill switch)
-- - `bot_state`: heartbeat + last error/action
-- - `bot_events`: append-only audit log
-CREATE TABLE IF NOT EXISTS public.bot_config (
-  id UUID PRIMARY KEY DEFAULT gen_random_uuid(),
-  name TEXT NOT NULL UNIQUE,
-  market_name TEXT NOT NULL,
-  market_type TEXT NOT NULL DEFAULT 'perp',
-  mode TEXT NOT NULL DEFAULT 'off',
-  kill_switch BOOLEAN NOT NULL DEFAULT FALSE,
-  params JSONB,
-  created_at TIMESTAMPTZ NOT NULL DEFAULT now(),
-  updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
-);
-
-CREATE INDEX IF NOT EXISTS bot_config_updated_at_desc_idx
-  ON public.bot_config (updated_at DESC);
-
-CREATE TABLE IF NOT EXISTS public.bot_state (
-  bot_id UUID PRIMARY KEY REFERENCES public.bot_config(id) ON DELETE CASCADE,
-  last_heartbeat_at TIMESTAMPTZ,
-  last_action_at TIMESTAMPTZ,
-  last_error TEXT,
-  state JSONB,
-  updated_at TIMESTAMPTZ NOT NULL DEFAULT now()
-);
-
-CREATE INDEX IF NOT EXISTS bot_state_updated_at_desc_idx
-  ON public.bot_state (updated_at DESC);
-
-CREATE TABLE IF NOT EXISTS public.bot_events (
-  id BIGSERIAL PRIMARY KEY,
-  bot_id UUID NOT NULL REFERENCES public.bot_config(id) ON DELETE CASCADE,
-  ts TIMESTAMPTZ NOT NULL DEFAULT now(),
-  type TEXT NOT NULL,
-  payload JSONB
-);
-
-CREATE INDEX IF NOT EXISTS bot_events_bot_ts_desc_idx
-  ON public.bot_events (bot_id, ts DESC);
-
 -- Compute OHLC candles from `drift_ticks` for a symbol and bucket size.
 -- Exposed via Hasura (track function) and used by trade-api to compute indicators server-side.
 -- Hasura tracks functions only if they return SETOF a table/view type.
@@ -858,3 +814,246 @@ BEGIN
  PERFORM add_retention_policy('dlob_slippage_ts_v2', INTERVAL '7 days');
 EXCEPTION WHEN OTHERS THEN
 END $$;
+
+-- Canonical DLOB storage and projection layer.
+CREATE TABLE IF NOT EXISTS public.dlob_orderbook_l2_latest (
+  source TEXT NOT NULL,
+  market_name TEXT NOT NULL,
+  market_type TEXT NOT NULL,
+  market_index INTEGER,
+  is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
+  ts BIGINT,
+  slot BIGINT,
+  mark_price_raw TEXT,
+  oracle_price_raw TEXT,
+  best_bid_price_raw TEXT,
+  best_ask_price_raw TEXT,
+  spread_pct_raw TEXT,
+  spread_quote_raw TEXT,
+  oracle_data JSONB,
+  bids JSONB,
+  asks JSONB,
+  payload JSONB NOT NULL,
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  PRIMARY KEY (source, market_name, is_indicative)
+);
+
+CREATE TABLE IF NOT EXISTS public.dlob_orderbook_l3_latest (
+  source TEXT NOT NULL,
+  market_name TEXT NOT NULL,
+  market_type TEXT NOT NULL,
+  market_index INTEGER,
+  is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
+  ts BIGINT,
+  slot BIGINT,
+  oracle_data JSONB,
+  bids JSONB,
+  asks JSONB,
+  payload JSONB NOT NULL,
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  PRIMARY KEY (source, market_name, is_indicative)
+);
+
+CREATE TABLE IF NOT EXISTS public.dlob_best_makers_latest (
+  source TEXT NOT NULL,
+  market_name TEXT NOT NULL,
+  market_type TEXT NOT NULL,
+  market_index INTEGER,
+  slot BIGINT,
+  bids JSONB,
+  asks JSONB,
+  payload JSONB NOT NULL,
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+  PRIMARY KEY (source, market_name)
+);
+
+CREATE INDEX IF NOT EXISTS dlob_orderbook_l2_latest_updated_at_idx
+  ON public.dlob_orderbook_l2_latest (updated_at DESC);
+CREATE INDEX IF NOT EXISTS dlob_orderbook_l3_latest_updated_at_idx
+  ON public.dlob_orderbook_l3_latest (updated_at DESC);
+CREATE INDEX IF NOT EXISTS dlob_best_makers_latest_updated_at_idx
+  ON public.dlob_best_makers_latest (updated_at DESC);
+
+CREATE OR REPLACE VIEW public.dlob_l2_latest_projection AS
+SELECT
+  source,
+  market_name,
+  market_type,
+  market_index,
+  ts,
+  slot,
+  CASE
+    WHEN NULLIF(mark_price_raw, '') IS NULL THEN NULL
+    ELSE NULLIF(mark_price_raw, '')::numeric / 1000000::numeric
+  END AS mark_price,
+  CASE
+    WHEN NULLIF(oracle_price_raw, '') IS NULL THEN NULL
+    ELSE NULLIF(oracle_price_raw, '')::numeric / 1000000::numeric
+  END AS oracle_price,
+  CASE
+    WHEN NULLIF(best_bid_price_raw, '') IS NULL THEN NULL
+    ELSE NULLIF(best_bid_price_raw, '')::numeric / 1000000::numeric
+  END AS best_bid_price,
+  CASE
+    WHEN NULLIF(best_ask_price_raw, '') IS NULL THEN NULL
+    ELSE NULLIF(best_ask_price_raw, '')::numeric / 1000000::numeric
+  END AS best_ask_price,
+  bids,
+  asks,
+  payload AS raw,
+  updated_at
+FROM public.dlob_orderbook_l2_latest
+WHERE is_indicative = FALSE;
+
+CREATE OR REPLACE VIEW public.dlob_l3_latest_projection AS
+SELECT
+  source,
+  market_name,
+  market_type,
+  market_index,
+  ts,
+  slot,
+  bids,
+  asks,
+  payload AS raw,
+  updated_at
+FROM public.dlob_orderbook_l3_latest
+WHERE is_indicative = FALSE;
+
+CREATE OR REPLACE VIEW public.dlob_best_makers_latest_projection AS
+SELECT
+  source,
+  market_name,
+  market_type,
+  market_index,
+  slot,
+  bids,
+  asks,
+  payload AS raw,
+  updated_at
+FROM public.dlob_best_makers_latest;
+
+CREATE OR REPLACE VIEW public.dlob_stats_latest_projection AS
+WITH base AS (
+  SELECT
+    source,
+    market_name,
+    market_type,
+    market_index,
+    ts,
+    slot,
+    CASE
+      WHEN NULLIF(mark_price_raw, '') IS NULL THEN NULL
+      ELSE NULLIF(mark_price_raw, '')::numeric / 1000000::numeric
+    END AS mark_price,
+    CASE
+      WHEN NULLIF(oracle_price_raw, '') IS NULL THEN NULL
+      ELSE NULLIF(oracle_price_raw, '')::numeric / 1000000::numeric
+    END AS oracle_price,
+    CASE
+      WHEN NULLIF(best_bid_price_raw, '') IS NULL THEN NULL
+      ELSE NULLIF(best_bid_price_raw, '')::numeric / 1000000::numeric
+    END AS best_bid_price,
+    CASE
+      WHEN NULLIF(best_ask_price_raw, '') IS NULL THEN NULL
+      ELSE NULLIF(best_ask_price_raw, '')::numeric / 1000000::numeric
+    END AS best_ask_price,
+    bids,
+    asks,
+    payload,
+    updated_at
+  FROM public.dlob_orderbook_l2_latest
+  WHERE is_indicative = FALSE
+),
+bid_levels AS (
+  SELECT
+    b.source,
+    b.market_name,
+    COALESCE(
+      SUM((NULLIF(level->>'size', '')::numeric) / 1000000000::numeric),
+      0::numeric
+    ) AS depth_bid_base,
+    COALESCE(
+      SUM(
+        ((NULLIF(level->>'size', '')::numeric) / 1000000000::numeric) *
+        ((NULLIF(level->>'price', '')::numeric) / 1000000::numeric)
+      ),
+      0::numeric
+    ) AS depth_bid_usd
+  FROM base b
+  LEFT JOIN LATERAL (
+    SELECT level
+    FROM jsonb_array_elements(COALESCE(b.bids, '[]'::jsonb)) WITH ORDINALITY AS e(level, ord)
+    WHERE ord <= 10
+  ) levels ON TRUE
+  GROUP BY b.source, b.market_name
+),
+ask_levels AS (
+  SELECT
+    b.source,
+    b.market_name,
+    COALESCE(
+      SUM((NULLIF(level->>'size', '')::numeric) / 1000000000::numeric),
+      0::numeric
+    ) AS depth_ask_base,
+    COALESCE(
+      SUM(
+        ((NULLIF(level->>'size', '')::numeric) / 1000000000::numeric) *
+        ((NULLIF(level->>'price', '')::numeric) / 1000000::numeric)
+      ),
+      0::numeric
+    ) AS depth_ask_usd
+  FROM base b
+  LEFT JOIN LATERAL (
+    SELECT level
+    FROM jsonb_array_elements(COALESCE(b.asks, '[]'::jsonb)) WITH ORDINALITY AS e(level, ord)
+    WHERE ord <= 10
+  ) levels ON TRUE
+  GROUP BY b.source, b.market_name
+)
+SELECT
+  b.source,
+  b.market_name,
+  b.market_type,
+  b.market_index,
+  b.ts,
+  b.slot,
+  b.mark_price,
+  b.oracle_price,
+  b.best_bid_price,
+  b.best_ask_price,
+  CASE
+    WHEN b.best_bid_price IS NOT NULL AND b.best_ask_price IS NOT NULL
+    THEN (b.best_bid_price + b.best_ask_price) / 2
+    ELSE NULL
+  END AS mid_price,
+  CASE
+    WHEN b.best_bid_price IS NOT NULL AND b.best_ask_price IS NOT NULL
+    THEN b.best_ask_price - b.best_bid_price
+    ELSE NULL
+  END AS spread_abs,
+  CASE
+    WHEN b.best_bid_price IS NOT NULL
+      AND b.best_ask_price IS NOT NULL
+      AND (b.best_bid_price + b.best_ask_price) > 0
+    THEN ((b.best_ask_price - b.best_bid_price) / ((b.best_bid_price + b.best_ask_price) / 2)) * 10000
+    ELSE NULL
+  END AS spread_bps,
+  10::integer AS depth_levels,
+  bids.depth_bid_base,
+  asks.depth_ask_base,
+  bids.depth_bid_usd,
+  asks.depth_ask_usd,
+  CASE
+    WHEN (bids.depth_bid_usd + asks.depth_ask_usd) > 0
+    THEN (bids.depth_bid_usd - asks.depth_ask_usd) / (bids.depth_bid_usd + asks.depth_ask_usd)
+    ELSE NULL
+  END AS imbalance,
+  jsonb_build_object(
+    'canonical_table', 'dlob_orderbook_l2_latest',
+    'projection', 'dlob_stats_latest_projection'
+  ) AS raw,
+  b.updated_at
+FROM base b
+LEFT JOIN bid_levels bids USING (source, market_name)
+LEFT JOIN ask_levels asks USING (source, market_name);
--- a/kustomize/overlays/mevnode-bot/README.md
+++ b/kustomize/overlays/mevnode-bot/README.md
@@ -0,0 +1,49 @@
+# mevnode-bot overlay
+
+To jest docelowy overlay dla runtime `mevnode_bot`.
+
+Zakres:
+
+- `postgres`
+- `hasura`
+- `trade-api`
+- `trade-frontend` (`NodePort 30081`)
+- `trade-ingestor`
+- `dlob-redis`
+- `dlob-publisher`
+- `dlob-server`
+- `dlob-worker`
+- `dlob-depth-worker`
+- `dlob-slippage-worker`
+
+Założenia runtime:
+
+- `mevnode_sol` wystawia po `wg0`:
+  - `RPC`: `http://10.91.0.1:8899`
+  - `WS`: `ws://10.91.0.1:8900`
+  - `Yellowstone`: `http://10.91.0.1:10000`
+- `mevnode_bot` uruchamia tylko warstwę aplikacyjną.
+- `trade-frontend` jest wystawiany przez `NodePort 30081`.
+- Namespace pozostaje `trade-staging`.
+
+Wymagane sekrety:
+
+- `trade-postgres`
+- `trade-hasura`
+- `trade-api`
+- `trade-frontend-tokens`
+- `trade-dlob-rpc`
+
+Opcjonalne:
+
+- `gitea-registry` nie jest tu wymagane, bo overlay czyści `imagePullSecrets`.
+
+Uwagi:
+
+- Overlay zastępuje lokalny `_tmp/bot-rx-stack`.
+- `trade-deploy/kustomize/overlays/mevnode-bot` jest teraz źródłem prawdy dla runtime `mevnode_bot`.
+- Ten overlay nadal korzysta z plików z `../../base` w `configMapGenerator`, więc przy lokalnym `kustomize` może być potrzebne:
+
+```bash
+sudo k3s kubectl kustomize trade-deploy/kustomize/overlays/mevnode-bot --load-restrictor LoadRestrictionsNone | sudo k3s kubectl apply -f -
+```
--- a/kustomize/overlays/mevnode-bot/dlob-ingestor.mjs
+++ b/kustomize/overlays/mevnode-bot/dlob-ingestor.mjs
@@ -0,0 +1,219 @@
+import process from 'node:process';
+import { setTimeout as sleep } from 'node:timers/promises';
+
+function getIsoNow() {
+  return new Date().toISOString();
+}
+
+function envString(name, fallback) {
+  const v = process.env[name];
+  if (v == null) return fallback;
+  const s = String(v).trim();
+  return s ? s : fallback;
+}
+
+function envInt(name, fallback, { min, max } = {}) {
+  const v = process.env[name];
+  if (v == null) return fallback;
+  const n = Number.parseInt(String(v), 10);
+  if (!Number.isFinite(n)) return fallback;
+  const low = typeof min === 'number' ? min : n;
+  const high = typeof max === 'number' ? max : n;
+  return Math.max(low, Math.min(high, n));
+}
+
+function envList(name, fallbackCsv) {
+  const raw = process.env[name] ?? fallbackCsv;
+  return String(raw)
+    .split(',')
+    .map((s) => s.trim())
+    .filter(Boolean);
+}
+
+function toIntOrNull(v) {
+  if (v == null) return null;
+  if (typeof v === 'number') return Number.isFinite(v) ? Math.trunc(v) : null;
+  if (typeof v === 'string') {
+    const s = v.trim();
+    if (!s) return null;
+    const n = Number.parseInt(s, 10);
+    return Number.isFinite(n) ? n : null;
+  }
+  return null;
+}
+
+function numStr(v) {
+  if (v == null) return null;
+  if (typeof v === 'number') return Number.isFinite(v) ? String(v) : null;
+  if (typeof v === 'string') {
+    const s = v.trim();
+    return s ? s : null;
+  }
+  return null;
+}
+
+function isoFromEpochMs(v) {
+  const n = typeof v === 'number' ? v : typeof v === 'string' ? Number(v.trim()) : NaN;
+  if (!Number.isFinite(n) || n <= 0) return null;
+  const d = new Date(n);
+  const ms = d.getTime();
+  if (!Number.isFinite(ms)) return null;
+  return d.toISOString();
+}
+
+function resolveConfig() {
+  const hasuraUrl = envString('HASURA_GRAPHQL_URL', 'http://hasura:8080/v1/graphql');
+  const hasuraAdminSecret = envString('HASURA_ADMIN_SECRET', '');
+  if (!hasuraAdminSecret) throw new Error('Missing HASURA_ADMIN_SECRET');
+
+  const markets = envList('DLOB_MARKETS', 'SOL-PERP,PUMP-PERP');
+  const pollMs = envInt('TICKS_POLL_MS', 1000, { min: 250, max: 60000 });
+  const source = envString('TICKS_SOURCE', 'dlob_stats');
+
+  return { hasuraUrl, hasuraAdminSecret, markets, pollMs, source };
+}
+
+async function graphqlRequest(cfg, query, variables) {
+  const res = await fetch(cfg.hasuraUrl, {
+    method: 'POST',
+    headers: {
+      'content-type': 'application/json',
+      'x-hasura-admin-secret': cfg.hasuraAdminSecret,
+    },
+    body: JSON.stringify({ query, variables }),
+    signal: AbortSignal.timeout(10000),
+  });
+
+  const text = await res.text();
+  if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`);
+
+  let json;
+  try {
+    json = JSON.parse(text);
+  } catch {
+    throw new Error(`Hasura: invalid json: ${text}`);
+  }
+
+  if (json.errors?.length) throw new Error(json.errors.map((e) => e.message).join(' | '));
+  return json.data;
+}
+
+async function fetchStats(cfg) {
+  const query = `
+    query DlobStatsLatest($markets: [String!]!) {
+      dlob_stats_latest(where: { market_name: { _in: $markets } }) {
+        market_name
+        market_index
+        ts
+        slot
+        oracle_price
+        mark_price
+        mid_price
+        best_bid_price
+        best_ask_price
+        updated_at
+      }
+    }
+  `;
+
+  const data = await graphqlRequest(cfg, query, { markets: cfg.markets });
+  const rows = Array.isArray(data?.dlob_stats_latest) ? data.dlob_stats_latest : [];
+  return rows;
+}
+
+async function insertTicks(cfg, objects) {
+  if (!objects.length) return 0;
+
+  const mutation = `
+    mutation InsertTicks($objects: [drift_ticks_insert_input!]!) {
+      insert_drift_ticks(objects: $objects) { affected_rows }
+    }
+  `;
+
+  const data = await graphqlRequest(cfg, mutation, { objects });
+  return Number(data?.insert_drift_ticks?.affected_rows || 0);
+}
+
+async function main() {
+  const cfg = resolveConfig();
+  const lastUpdatedAtByMarket = new Map();
+
+  console.log(
+    JSON.stringify(
+      {
+        service: 'trade-ingestor',
+        mode: 'dlob_stats_ticks',
+        startedAt: getIsoNow(),
+        hasuraUrl: cfg.hasuraUrl,
+        markets: cfg.markets,
+        pollMs: cfg.pollMs,
+        source: cfg.source,
+      },
+      null,
+      2
+    )
+  );
+
+  while (true) {
+    try {
+      const rows = await fetchStats(cfg);
+      const nowIso = getIsoNow();
+
+      const objects = [];
+      for (const r of rows) {
+        const marketName = String(r?.market_name || '').trim();
+        if (!marketName) continue;
+
+        const updatedAt = r?.updated_at ? String(r.updated_at) : '';
+        if (updatedAt && lastUpdatedAtByMarket.get(marketName) === updatedAt) continue;
+        if (updatedAt) lastUpdatedAtByMarket.set(marketName, updatedAt);
+
+        const marketIndex = toIntOrNull(r?.market_index) ?? 0;
+        const dlobIso = isoFromEpochMs(r?.ts);
+        const tsIso = dlobIso || nowIso;
+
+        const oraclePrice = numStr(r?.oracle_price) || numStr(r?.mark_price) || numStr(r?.mid_price);
+        const markPrice = numStr(r?.mark_price) || numStr(r?.mid_price) || oraclePrice;
+        if (!oraclePrice) continue;
+
+        objects.push({
+          ts: tsIso,
+          market_index: marketIndex,
+          symbol: marketName,
+          oracle_price: oraclePrice,
+          mark_price: markPrice,
+          oracle_slot: r?.slot == null ? null : String(r.slot),
+          source: cfg.source,
+          raw: {
+            from: 'dlob_stats_latest',
+            market_name: marketName,
+            market_index: marketIndex,
+            dlob: {
+              ts: r?.ts ?? null,
+              slot: r?.slot ?? null,
+              best_bid_price: r?.best_bid_price ?? null,
+              best_ask_price: r?.best_ask_price ?? null,
+              mid_price: r?.mid_price ?? null,
+              updated_at: updatedAt || null,
+            },
+          },
+        });
+      }
+
+      const inserted = await insertTicks(cfg, objects);
+      if (inserted) {
+        console.log(`[dlob-ticks] inserted=${inserted} ts=${nowIso}`);
+      }
+    } catch (err) {
+      console.error(`[dlob-ticks] error: ${String(err?.message || err)}`);
+      await sleep(2000);
+    }
+
+    await sleep(cfg.pollMs);
+  }
+}
+
+main().catch((err) => {
+  console.error(String(err?.stack || err));
+  process.exitCode = 1;
+});
--- a/kustomize/overlays/mevnode-bot/kustomization.yaml
+++ b/kustomize/overlays/mevnode-bot/kustomization.yaml
@@ -0,0 +1,67 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: trade-staging
+
+resources:
+  - namespace.yaml
+  - ../../base/cm-trade-deploy.yaml
+  - ../../base/postgres/service.yaml
+  - ../../base/postgres/statefulset.yaml
+  - ../../base/postgres/job-migrate.yaml
+  - ../../base/hasura/service.yaml
+  - ../../base/hasura/deployment.yaml
+  - ../../base/hasura/job-bootstrap.yaml
+  - ../../base/api/service.yaml
+  - ../../base/api/deployment.yaml
+  - ../../base/ingestor/deployment.yaml
+  - ../../base/frontend/service.yaml
+  - ../../base/frontend/deployment.yaml
+  - ../../base/dlob/redis.yaml
+  - ../../base/dlob/publisher-deployment.yaml
+  - ../../base/dlob/server-service.yaml
+  - ../../base/dlob/server-deployment.yaml
+  - ../../base/dlob-worker/deployment.yaml
+  - ../../base/dlob-depth-worker/deployment.yaml
+  - ../../base/dlob-slippage-worker/deployment.yaml
+
+patchesStrategicMerge:
+  - patch-api.yaml
+  - patch-frontend.yaml
+  - patch-frontend-service.yaml
+  - patch-ingestor.yaml
+  - patch-dlob-publisher.yaml
+  - patch-dlob-publisher-storage.yaml
+  - patch-dlob-server.yaml
+  - patch-dlob-worker.yaml
+  - patch-dlob-depth-worker.yaml
+  - patch-dlob-slippage-worker.yaml
+
+configMapGenerator:
+  - name: postgres-initdb
+    files:
+      - ../../base/initdb/001_init.sql
+  - name: hasura-bootstrap-script
+    files:
+      - ../../base/hasura/hasura-bootstrap.mjs
+  - name: trade-api-wrapper
+    files:
+      - ../../base/api/wrapper.mjs
+  - name: trade-api-upstream
+    files:
+      - ../../base/api/server.mjs
+  - name: dlob-worker-script
+    files:
+      - ../../base/dlob-worker/worker.mjs
+  - name: dlob-depth-worker-script
+    files:
+      - ../../base/dlob-depth-worker/worker.mjs
+  - name: dlob-slippage-worker-script
+    files:
+      - ../../base/dlob-slippage-worker/worker.mjs
+  - name: trade-dlob-ingestor-script
+    files:
+      - dlob-ingestor.mjs
+
+generatorOptions:
+  disableNameSuffixHash: true
--- a/kustomize/overlays/mevnode-bot/namespace.yaml
+++ b/kustomize/overlays/mevnode-bot/namespace.yaml
@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: trade-staging
--- a/kustomize/overlays/mevnode-bot/patch-api.yaml
+++ b/kustomize/overlays/mevnode-bot/patch-api.yaml
@@ -0,0 +1,8 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: trade-api
+spec:
+  template:
+    spec:
+      imagePullSecrets: []
--- a/kustomize/overlays/mevnode-bot/patch-dlob-depth-worker.yaml
+++ b/kustomize/overlays/mevnode-bot/patch-dlob-depth-worker.yaml
@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dlob-depth-worker
+spec:
+  template:
+    spec:
+      containers:
+        - name: worker
+          env:
+            - name: DLOB_MARKETS
+              value: SOL-PERP,PUMP-PERP
--- a/kustomize/overlays/mevnode-bot/patch-dlob-publisher-storage.yaml
+++ b/kustomize/overlays/mevnode-bot/patch-dlob-publisher-storage.yaml
@@ -0,0 +1,40 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dlob-publisher
+spec:
+  replicas: 1
+  template:
+    spec:
+      containers:
+        - name: publisher
+          env:
+            - name: ENABLE_PERSISTENT_STORE
+              value: "true"
+            - name: DLOB_SOURCE
+              value: "mevnode_bot"
+            - name: PRICE_PRECISION
+              value: "1000000"
+            - name: BASE_PRECISION
+              value: "1000000000"
+            - name: PERSISTENT_STATS_DEPTH
+              value: "10"
+            - name: PGHOST
+              value: postgres
+            - name: PGPORT
+              value: "5432"
+            - name: PGUSER
+              valueFrom:
+                secretKeyRef:
+                  name: trade-postgres
+                  key: POSTGRES_USER
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: trade-postgres
+                  key: POSTGRES_PASSWORD
+            - name: PGDATABASE
+              valueFrom:
+                secretKeyRef:
+                  name: trade-postgres
+                  key: POSTGRES_DB
--- a/kustomize/overlays/mevnode-bot/patch-dlob-publisher.yaml
+++ b/kustomize/overlays/mevnode-bot/patch-dlob-publisher.yaml
@@ -0,0 +1,31 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dlob-publisher
+spec:
+  template:
+    spec:
+      imagePullSecrets: []
+      containers:
+        - name: publisher
+          env:
+            - name: PERP_MARKETS_TO_LOAD
+              value: "0,75"
+            - name: USE_GRPC
+              value: "true"
+            - name: USE_WEBSOCKET
+              value: "true"
+            - name: DISABLE_GPA_REFRESH
+              value: "true"
+            - name: GRPC_CLIENT
+              value: "yellowstone"
+            - name: GRPC_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: trade-dlob-rpc
+                  key: GRPC_ENDPOINT
+            - name: TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: trade-dlob-rpc
+                  key: TOKEN
--- a/kustomize/overlays/mevnode-bot/patch-dlob-server.yaml
+++ b/kustomize/overlays/mevnode-bot/patch-dlob-server.yaml
@@ -0,0 +1,8 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dlob-server
+spec:
+  template:
+    spec:
+      imagePullSecrets: []
--- a/kustomize/overlays/mevnode-bot/patch-dlob-slippage-worker.yaml
+++ b/kustomize/overlays/mevnode-bot/patch-dlob-slippage-worker.yaml
@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dlob-slippage-worker
+spec:
+  template:
+    spec:
+      containers:
+        - name: worker
+          env:
+            - name: DLOB_MARKETS
+              value: SOL-PERP,PUMP-PERP
--- a/kustomize/overlays/mevnode-bot/patch-dlob-worker.yaml
+++ b/kustomize/overlays/mevnode-bot/patch-dlob-worker.yaml
@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dlob-worker
+spec:
+  template:
+    spec:
+      containers:
+        - name: worker
+          env:
+            - name: DLOB_MARKETS
+              value: SOL-PERP,PUMP-PERP
--- a/kustomize/overlays/mevnode-bot/patch-frontend-service.yaml
+++ b/kustomize/overlays/mevnode-bot/patch-frontend-service.yaml
@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: trade-frontend
+spec:
+  type: NodePort
+  ports:
+    - name: http
+      port: 8081
+      targetPort: http
+      nodePort: 30081
--- a/kustomize/overlays/mevnode-bot/patch-frontend.yaml
+++ b/kustomize/overlays/mevnode-bot/patch-frontend.yaml
@@ -0,0 +1,8 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: trade-frontend
+spec:
+  template:
+    spec:
+      imagePullSecrets: []
--- a/kustomize/overlays/mevnode-bot/patch-ingestor.yaml
+++ b/kustomize/overlays/mevnode-bot/patch-ingestor.yaml
@@ -0,0 +1,39 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: trade-ingestor
+spec:
+  template:
+    spec:
+      imagePullSecrets: []
+      containers:
+        - name: ingestor
+          image: node:20-slim
+          imagePullPolicy: IfNotPresent
+          env:
+            - name: HASURA_GRAPHQL_URL
+              value: http://hasura:8080/v1/graphql
+            - name: HASURA_ADMIN_SECRET
+              valueFrom:
+                secretKeyRef:
+                  name: trade-hasura
+                  key: HASURA_GRAPHQL_ADMIN_SECRET
+            - name: DLOB_MARKETS
+              value: SOL-PERP,PUMP-PERP
+            - name: TICKS_POLL_MS
+              value: "1000"
+            - name: TICKS_SOURCE
+              value: "dlob_stats"
+          command:
+            - node
+          args:
+            - /opt/dlob/dlob-ingestor.mjs
+          volumeMounts:
+            - name: dlob-script
+              mountPath: /opt/dlob/dlob-ingestor.mjs
+              subPath: dlob-ingestor.mjs
+              readOnly: true
+      volumes:
+        - name: dlob-script
+          configMap:
+            name: trade-dlob-ingestor-script
--- a/kustomize/overlays/prod/dlob-rpc-endpoint-patch.yaml
+++ b/kustomize/overlays/prod/dlob-rpc-endpoint-patch.yaml
@@ -1,14 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: dlob-publisher
-spec:
-  template:
-    spec:
-      containers:
-        - name: publisher
-          env:
-            - name: ENDPOINT
-              value: "http://10.66.66.1:8899"
-            - name: WS_ENDPOINT
-              value: "ws://10.66.66.1:8900"
--- a/kustomize/overlays/prod/dlob-rpc-server-endpoint-patch.yaml
+++ b/kustomize/overlays/prod/dlob-rpc-server-endpoint-patch.yaml
@@ -1,14 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: dlob-server
-spec:
-  template:
-    spec:
-      containers:
-        - name: server
-          env:
-            - name: ENDPOINT
-              value: "http://10.66.66.1:8899"
-            - name: WS_ENDPOINT
-              value: "ws://10.66.66.1:8900"
--- a/kustomize/overlays/prod/kustomization.yaml
+++ b/kustomize/overlays/prod/kustomization.yaml
@@ -7,8 +7,6 @@ resources:
  - ../../base

 patchesStrategicMerge:
-  - dlob-rpc-endpoint-patch.yaml
-  - dlob-rpc-server-endpoint-patch.yaml
  - frontend-graphql-proxy-patch.yaml

 configMapGenerator:
--- a/kustomize/overlays/staging/dlob-rpc-endpoint-patch.yaml
+++ b/kustomize/overlays/staging/dlob-rpc-endpoint-patch.yaml
@@ -1,14 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: dlob-publisher
-spec:
-  template:
-    spec:
-      containers:
-        - name: publisher
-          env:
-            - name: ENDPOINT
-              value: "http://10.66.66.1:8899"
-            - name: WS_ENDPOINT
-              value: "ws://10.66.66.1:8900"
--- a/kustomize/overlays/staging/dlob-rpc-server-endpoint-patch.yaml
+++ b/kustomize/overlays/staging/dlob-rpc-server-endpoint-patch.yaml
@@ -1,14 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: dlob-server
-spec:
-  template:
-    spec:
-      containers:
-        - name: server
-          env:
-            - name: ENDPOINT
-              value: "http://10.66.66.1:8899"
-            - name: WS_ENDPOINT
-              value: "ws://10.66.66.1:8900"
--- a/kustomize/overlays/staging/kustomization.yaml
+++ b/kustomize/overlays/staging/kustomization.yaml
@@ -10,8 +10,6 @@ resources:
  - frontend-ingress-root.yaml

 patchesStrategicMerge:
-  - dlob-rpc-endpoint-patch.yaml
-  - dlob-rpc-server-endpoint-patch.yaml
  - hasura-patch.yaml
  - frontend-auth-patch.yaml
  - frontend-graphql-proxy-patch.yaml
Author	SHA1	Message	Date
mpabi	71b00f80bb	feat(hasura): expose canonical dlob projections	2026-03-13 23:42:18 +01:00
mpabi	01f6543ae9	chore(dlob): enable publisher with canonical store	2026-03-13 23:39:30 +01:00
mpabi	8a474242df	chore(dlob): enable websocket feed on bot overlay	2026-03-13 23:32:20 +01:00
mpabi	db4e45dc55	chore(dlob): use canonical storage image	2026-03-13 23:28:18 +01:00
mpabi	3572382c90	feat(dlob): prepare canonical storage bootstrap	2026-03-13 23:18:00 +01:00
mpabi	bb6f633d19	feat(kustomize): add mevnode bot overlay	2026-03-13 22:33:28 +01:00
mpabi	108bf22968	chore(kustomize): remove dlob rpc endpoint patches	2026-03-12 23:26:17 +01:00