From 3572382c90adb255116db8a991ca3bbc1d80f7b9 Mon Sep 17 00:00:00 2001 From: mpabi Date: Fri, 13 Mar 2026 23:18:00 +0100 Subject: [PATCH] feat(dlob): prepare canonical storage bootstrap --- kustomize/base/initdb/001_init.sql | 243 ++++++++++++++++++ .../overlays/mevnode-bot/kustomization.yaml | 1 + .../patch-dlob-publisher-storage.yaml | 40 +++ 3 files changed, 284 insertions(+) create mode 100644 kustomize/overlays/mevnode-bot/patch-dlob-publisher-storage.yaml diff --git a/kustomize/base/initdb/001_init.sql b/kustomize/base/initdb/001_init.sql index 6618671..70f9f19 100644 --- a/kustomize/base/initdb/001_init.sql +++ b/kustomize/base/initdb/001_init.sql @@ -814,3 +814,246 @@ BEGIN PERFORM add_retention_policy('dlob_slippage_ts_v2', INTERVAL '7 days'); EXCEPTION WHEN OTHERS THEN END $$; + +-- Canonical DLOB storage and projection layer. +CREATE TABLE IF NOT EXISTS public.dlob_orderbook_l2_latest ( + source TEXT NOT NULL, + market_name TEXT NOT NULL, + market_type TEXT NOT NULL, + market_index INTEGER, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts BIGINT, + slot BIGINT, + mark_price_raw TEXT, + oracle_price_raw TEXT, + best_bid_price_raw TEXT, + best_ask_price_raw TEXT, + spread_pct_raw TEXT, + spread_quote_raw TEXT, + oracle_data JSONB, + bids JSONB, + asks JSONB, + payload JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (source, market_name, is_indicative) +); + +CREATE TABLE IF NOT EXISTS public.dlob_orderbook_l3_latest ( + source TEXT NOT NULL, + market_name TEXT NOT NULL, + market_type TEXT NOT NULL, + market_index INTEGER, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts BIGINT, + slot BIGINT, + oracle_data JSONB, + bids JSONB, + asks JSONB, + payload JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (source, market_name, is_indicative) +); + +CREATE TABLE IF NOT EXISTS public.dlob_best_makers_latest ( + source TEXT NOT NULL, + market_name TEXT NOT NULL, + market_type TEXT NOT NULL, + market_index INTEGER, + slot BIGINT, + bids JSONB, + asks JSONB, + payload JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (source, market_name) +); + +CREATE INDEX IF NOT EXISTS dlob_orderbook_l2_latest_updated_at_idx + ON public.dlob_orderbook_l2_latest (updated_at DESC); +CREATE INDEX IF NOT EXISTS dlob_orderbook_l3_latest_updated_at_idx + ON public.dlob_orderbook_l3_latest (updated_at DESC); +CREATE INDEX IF NOT EXISTS dlob_best_makers_latest_updated_at_idx + ON public.dlob_best_makers_latest (updated_at DESC); + +CREATE OR REPLACE VIEW public.dlob_l2_latest_projection AS +SELECT + source, + market_name, + market_type, + market_index, + ts, + slot, + CASE + WHEN NULLIF(mark_price_raw, '') IS NULL THEN NULL + ELSE NULLIF(mark_price_raw, '')::numeric / 1000000::numeric + END AS mark_price, + CASE + WHEN NULLIF(oracle_price_raw, '') IS NULL THEN NULL + ELSE NULLIF(oracle_price_raw, '')::numeric / 1000000::numeric + END AS oracle_price, + CASE + WHEN NULLIF(best_bid_price_raw, '') IS NULL THEN NULL + ELSE NULLIF(best_bid_price_raw, '')::numeric / 1000000::numeric + END AS best_bid_price, + CASE + WHEN NULLIF(best_ask_price_raw, '') IS NULL THEN NULL + ELSE NULLIF(best_ask_price_raw, '')::numeric / 1000000::numeric + END AS best_ask_price, + bids, + asks, + payload AS raw, + updated_at +FROM public.dlob_orderbook_l2_latest +WHERE is_indicative = FALSE; + +CREATE OR REPLACE VIEW public.dlob_l3_latest_projection AS +SELECT + source, + market_name, + market_type, + market_index, + ts, + slot, + bids, + asks, + payload AS raw, + updated_at +FROM public.dlob_orderbook_l3_latest +WHERE is_indicative = FALSE; + +CREATE OR REPLACE VIEW public.dlob_best_makers_latest_projection AS +SELECT + source, + market_name, + market_type, + market_index, + slot, + bids, + asks, + payload AS raw, + updated_at +FROM public.dlob_best_makers_latest; + +CREATE OR REPLACE VIEW public.dlob_stats_latest_projection AS +WITH base AS ( + SELECT + source, + market_name, + market_type, + market_index, + ts, + slot, + CASE + WHEN NULLIF(mark_price_raw, '') IS NULL THEN NULL + ELSE NULLIF(mark_price_raw, '')::numeric / 1000000::numeric + END AS mark_price, + CASE + WHEN NULLIF(oracle_price_raw, '') IS NULL THEN NULL + ELSE NULLIF(oracle_price_raw, '')::numeric / 1000000::numeric + END AS oracle_price, + CASE + WHEN NULLIF(best_bid_price_raw, '') IS NULL THEN NULL + ELSE NULLIF(best_bid_price_raw, '')::numeric / 1000000::numeric + END AS best_bid_price, + CASE + WHEN NULLIF(best_ask_price_raw, '') IS NULL THEN NULL + ELSE NULLIF(best_ask_price_raw, '')::numeric / 1000000::numeric + END AS best_ask_price, + bids, + asks, + payload, + updated_at + FROM public.dlob_orderbook_l2_latest + WHERE is_indicative = FALSE +), +bid_levels AS ( + SELECT + b.source, + b.market_name, + COALESCE( + SUM((NULLIF(level->>'size', '')::numeric) / 1000000000::numeric), + 0::numeric + ) AS depth_bid_base, + COALESCE( + SUM( + ((NULLIF(level->>'size', '')::numeric) / 1000000000::numeric) * + ((NULLIF(level->>'price', '')::numeric) / 1000000::numeric) + ), + 0::numeric + ) AS depth_bid_usd + FROM base b + LEFT JOIN LATERAL ( + SELECT level + FROM jsonb_array_elements(COALESCE(b.bids, '[]'::jsonb)) WITH ORDINALITY AS e(level, ord) + WHERE ord <= 10 + ) levels ON TRUE + GROUP BY b.source, b.market_name +), +ask_levels AS ( + SELECT + b.source, + b.market_name, + COALESCE( + SUM((NULLIF(level->>'size', '')::numeric) / 1000000000::numeric), + 0::numeric + ) AS depth_ask_base, + COALESCE( + SUM( + ((NULLIF(level->>'size', '')::numeric) / 1000000000::numeric) * + ((NULLIF(level->>'price', '')::numeric) / 1000000::numeric) + ), + 0::numeric + ) AS depth_ask_usd + FROM base b + LEFT JOIN LATERAL ( + SELECT level + FROM jsonb_array_elements(COALESCE(b.asks, '[]'::jsonb)) WITH ORDINALITY AS e(level, ord) + WHERE ord <= 10 + ) levels ON TRUE + GROUP BY b.source, b.market_name +) +SELECT + b.source, + b.market_name, + b.market_type, + b.market_index, + b.ts, + b.slot, + b.mark_price, + b.oracle_price, + b.best_bid_price, + b.best_ask_price, + CASE + WHEN b.best_bid_price IS NOT NULL AND b.best_ask_price IS NOT NULL + THEN (b.best_bid_price + b.best_ask_price) / 2 + ELSE NULL + END AS mid_price, + CASE + WHEN b.best_bid_price IS NOT NULL AND b.best_ask_price IS NOT NULL + THEN b.best_ask_price - b.best_bid_price + ELSE NULL + END AS spread_abs, + CASE + WHEN b.best_bid_price IS NOT NULL + AND b.best_ask_price IS NOT NULL + AND (b.best_bid_price + b.best_ask_price) > 0 + THEN ((b.best_ask_price - b.best_bid_price) / ((b.best_bid_price + b.best_ask_price) / 2)) * 10000 + ELSE NULL + END AS spread_bps, + 10::integer AS depth_levels, + bids.depth_bid_base, + asks.depth_ask_base, + bids.depth_bid_usd, + asks.depth_ask_usd, + CASE + WHEN (bids.depth_bid_usd + asks.depth_ask_usd) > 0 + THEN (bids.depth_bid_usd - asks.depth_ask_usd) / (bids.depth_bid_usd + asks.depth_ask_usd) + ELSE NULL + END AS imbalance, + jsonb_build_object( + 'canonical_table', 'dlob_orderbook_l2_latest', + 'projection', 'dlob_stats_latest_projection' + ) AS raw, + b.updated_at +FROM base b +LEFT JOIN bid_levels bids USING (source, market_name) +LEFT JOIN ask_levels asks USING (source, market_name); diff --git a/kustomize/overlays/mevnode-bot/kustomization.yaml b/kustomize/overlays/mevnode-bot/kustomization.yaml index b4378c1..38987d0 100644 --- a/kustomize/overlays/mevnode-bot/kustomization.yaml +++ b/kustomize/overlays/mevnode-bot/kustomization.yaml @@ -31,6 +31,7 @@ patchesStrategicMerge: - patch-frontend-service.yaml - patch-ingestor.yaml - patch-dlob-publisher.yaml + - patch-dlob-publisher-storage.yaml - patch-dlob-server.yaml - patch-dlob-worker.yaml - patch-dlob-depth-worker.yaml diff --git a/kustomize/overlays/mevnode-bot/patch-dlob-publisher-storage.yaml b/kustomize/overlays/mevnode-bot/patch-dlob-publisher-storage.yaml new file mode 100644 index 0000000..af31077 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/patch-dlob-publisher-storage.yaml @@ -0,0 +1,40 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dlob-publisher +spec: + replicas: 0 + template: + spec: + containers: + - name: publisher + env: + - name: ENABLE_PERSISTENT_STORE + value: "true" + - name: DLOB_SOURCE + value: "mevnode_bot" + - name: PRICE_PRECISION + value: "1000000" + - name: BASE_PRECISION + value: "1000000000" + - name: PERSISTENT_STATS_DEPTH + value: "10" + - name: PGHOST + value: postgres + - name: PGPORT + value: "5432" + - name: PGUSER + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_PASSWORD + - name: PGDATABASE + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_DB