feat(dlob): prepare canonical storage bootstrap
This commit is contained in:
mpabi
@@ -814,3 +814,246 @@ BEGIN
|
|||||||
PERFORM add_retention_policy('dlob_slippage_ts_v2', INTERVAL '7 days');
|
PERFORM add_retention_policy('dlob_slippage_ts_v2', INTERVAL '7 days');
|
||||||
EXCEPTION WHEN OTHERS THEN
|
EXCEPTION WHEN OTHERS THEN
|
||||||
END $$;
|
END $$;
|
||||||
|
|
||||||
|
-- Canonical DLOB storage and projection layer.
|
||||||
|
CREATE TABLE IF NOT EXISTS public.dlob_orderbook_l2_latest (
|
||||||
|
source TEXT NOT NULL,
|
||||||
|
market_name TEXT NOT NULL,
|
||||||
|
market_type TEXT NOT NULL,
|
||||||
|
market_index INTEGER,
|
||||||
|
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
ts BIGINT,
|
||||||
|
slot BIGINT,
|
||||||
|
mark_price_raw TEXT,
|
||||||
|
oracle_price_raw TEXT,
|
||||||
|
best_bid_price_raw TEXT,
|
||||||
|
best_ask_price_raw TEXT,
|
||||||
|
spread_pct_raw TEXT,
|
||||||
|
spread_quote_raw TEXT,
|
||||||
|
oracle_data JSONB,
|
||||||
|
bids JSONB,
|
||||||
|
asks JSONB,
|
||||||
|
payload JSONB NOT NULL,
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
PRIMARY KEY (source, market_name, is_indicative)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS public.dlob_orderbook_l3_latest (
|
||||||
|
source TEXT NOT NULL,
|
||||||
|
market_name TEXT NOT NULL,
|
||||||
|
market_type TEXT NOT NULL,
|
||||||
|
market_index INTEGER,
|
||||||
|
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
|
||||||
|
ts BIGINT,
|
||||||
|
slot BIGINT,
|
||||||
|
oracle_data JSONB,
|
||||||
|
bids JSONB,
|
||||||
|
asks JSONB,
|
||||||
|
payload JSONB NOT NULL,
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
PRIMARY KEY (source, market_name, is_indicative)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS public.dlob_best_makers_latest (
|
||||||
|
source TEXT NOT NULL,
|
||||||
|
market_name TEXT NOT NULL,
|
||||||
|
market_type TEXT NOT NULL,
|
||||||
|
market_index INTEGER,
|
||||||
|
slot BIGINT,
|
||||||
|
bids JSONB,
|
||||||
|
asks JSONB,
|
||||||
|
payload JSONB NOT NULL,
|
||||||
|
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||||
|
PRIMARY KEY (source, market_name)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_orderbook_l2_latest_updated_at_idx
|
||||||
|
ON public.dlob_orderbook_l2_latest (updated_at DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_orderbook_l3_latest_updated_at_idx
|
||||||
|
ON public.dlob_orderbook_l3_latest (updated_at DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS dlob_best_makers_latest_updated_at_idx
|
||||||
|
ON public.dlob_best_makers_latest (updated_at DESC);
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW public.dlob_l2_latest_projection AS
|
||||||
|
SELECT
|
||||||
|
source,
|
||||||
|
market_name,
|
||||||
|
market_type,
|
||||||
|
market_index,
|
||||||
|
ts,
|
||||||
|
slot,
|
||||||
|
CASE
|
||||||
|
WHEN NULLIF(mark_price_raw, '') IS NULL THEN NULL
|
||||||
|
ELSE NULLIF(mark_price_raw, '')::numeric / 1000000::numeric
|
||||||
|
END AS mark_price,
|
||||||
|
CASE
|
||||||
|
WHEN NULLIF(oracle_price_raw, '') IS NULL THEN NULL
|
||||||
|
ELSE NULLIF(oracle_price_raw, '')::numeric / 1000000::numeric
|
||||||
|
END AS oracle_price,
|
||||||
|
CASE
|
||||||
|
WHEN NULLIF(best_bid_price_raw, '') IS NULL THEN NULL
|
||||||
|
ELSE NULLIF(best_bid_price_raw, '')::numeric / 1000000::numeric
|
||||||
|
END AS best_bid_price,
|
||||||
|
CASE
|
||||||
|
WHEN NULLIF(best_ask_price_raw, '') IS NULL THEN NULL
|
||||||
|
ELSE NULLIF(best_ask_price_raw, '')::numeric / 1000000::numeric
|
||||||
|
END AS best_ask_price,
|
||||||
|
bids,
|
||||||
|
asks,
|
||||||
|
payload AS raw,
|
||||||
|
updated_at
|
||||||
|
FROM public.dlob_orderbook_l2_latest
|
||||||
|
WHERE is_indicative = FALSE;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW public.dlob_l3_latest_projection AS
|
||||||
|
SELECT
|
||||||
|
source,
|
||||||
|
market_name,
|
||||||
|
market_type,
|
||||||
|
market_index,
|
||||||
|
ts,
|
||||||
|
slot,
|
||||||
|
bids,
|
||||||
|
asks,
|
||||||
|
payload AS raw,
|
||||||
|
updated_at
|
||||||
|
FROM public.dlob_orderbook_l3_latest
|
||||||
|
WHERE is_indicative = FALSE;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW public.dlob_best_makers_latest_projection AS
|
||||||
|
SELECT
|
||||||
|
source,
|
||||||
|
market_name,
|
||||||
|
market_type,
|
||||||
|
market_index,
|
||||||
|
slot,
|
||||||
|
bids,
|
||||||
|
asks,
|
||||||
|
payload AS raw,
|
||||||
|
updated_at
|
||||||
|
FROM public.dlob_best_makers_latest;
|
||||||
|
|
||||||
|
CREATE OR REPLACE VIEW public.dlob_stats_latest_projection AS
|
||||||
|
WITH base AS (
|
||||||
|
SELECT
|
||||||
|
source,
|
||||||
|
market_name,
|
||||||
|
market_type,
|
||||||
|
market_index,
|
||||||
|
ts,
|
||||||
|
slot,
|
||||||
|
CASE
|
||||||
|
WHEN NULLIF(mark_price_raw, '') IS NULL THEN NULL
|
||||||
|
ELSE NULLIF(mark_price_raw, '')::numeric / 1000000::numeric
|
||||||
|
END AS mark_price,
|
||||||
|
CASE
|
||||||
|
WHEN NULLIF(oracle_price_raw, '') IS NULL THEN NULL
|
||||||
|
ELSE NULLIF(oracle_price_raw, '')::numeric / 1000000::numeric
|
||||||
|
END AS oracle_price,
|
||||||
|
CASE
|
||||||
|
WHEN NULLIF(best_bid_price_raw, '') IS NULL THEN NULL
|
||||||
|
ELSE NULLIF(best_bid_price_raw, '')::numeric / 1000000::numeric
|
||||||
|
END AS best_bid_price,
|
||||||
|
CASE
|
||||||
|
WHEN NULLIF(best_ask_price_raw, '') IS NULL THEN NULL
|
||||||
|
ELSE NULLIF(best_ask_price_raw, '')::numeric / 1000000::numeric
|
||||||
|
END AS best_ask_price,
|
||||||
|
bids,
|
||||||
|
asks,
|
||||||
|
payload,
|
||||||
|
updated_at
|
||||||
|
FROM public.dlob_orderbook_l2_latest
|
||||||
|
WHERE is_indicative = FALSE
|
||||||
|
),
|
||||||
|
bid_levels AS (
|
||||||
|
SELECT
|
||||||
|
b.source,
|
||||||
|
b.market_name,
|
||||||
|
COALESCE(
|
||||||
|
SUM((NULLIF(level->>'size', '')::numeric) / 1000000000::numeric),
|
||||||
|
0::numeric
|
||||||
|
) AS depth_bid_base,
|
||||||
|
COALESCE(
|
||||||
|
SUM(
|
||||||
|
((NULLIF(level->>'size', '')::numeric) / 1000000000::numeric) *
|
||||||
|
((NULLIF(level->>'price', '')::numeric) / 1000000::numeric)
|
||||||
|
),
|
||||||
|
0::numeric
|
||||||
|
) AS depth_bid_usd
|
||||||
|
FROM base b
|
||||||
|
LEFT JOIN LATERAL (
|
||||||
|
SELECT level
|
||||||
|
FROM jsonb_array_elements(COALESCE(b.bids, '[]'::jsonb)) WITH ORDINALITY AS e(level, ord)
|
||||||
|
WHERE ord <= 10
|
||||||
|
) levels ON TRUE
|
||||||
|
GROUP BY b.source, b.market_name
|
||||||
|
),
|
||||||
|
ask_levels AS (
|
||||||
|
SELECT
|
||||||
|
b.source,
|
||||||
|
b.market_name,
|
||||||
|
COALESCE(
|
||||||
|
SUM((NULLIF(level->>'size', '')::numeric) / 1000000000::numeric),
|
||||||
|
0::numeric
|
||||||
|
) AS depth_ask_base,
|
||||||
|
COALESCE(
|
||||||
|
SUM(
|
||||||
|
((NULLIF(level->>'size', '')::numeric) / 1000000000::numeric) *
|
||||||
|
((NULLIF(level->>'price', '')::numeric) / 1000000::numeric)
|
||||||
|
),
|
||||||
|
0::numeric
|
||||||
|
) AS depth_ask_usd
|
||||||
|
FROM base b
|
||||||
|
LEFT JOIN LATERAL (
|
||||||
|
SELECT level
|
||||||
|
FROM jsonb_array_elements(COALESCE(b.asks, '[]'::jsonb)) WITH ORDINALITY AS e(level, ord)
|
||||||
|
WHERE ord <= 10
|
||||||
|
) levels ON TRUE
|
||||||
|
GROUP BY b.source, b.market_name
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
b.source,
|
||||||
|
b.market_name,
|
||||||
|
b.market_type,
|
||||||
|
b.market_index,
|
||||||
|
b.ts,
|
||||||
|
b.slot,
|
||||||
|
b.mark_price,
|
||||||
|
b.oracle_price,
|
||||||
|
b.best_bid_price,
|
||||||
|
b.best_ask_price,
|
||||||
|
CASE
|
||||||
|
WHEN b.best_bid_price IS NOT NULL AND b.best_ask_price IS NOT NULL
|
||||||
|
THEN (b.best_bid_price + b.best_ask_price) / 2
|
||||||
|
ELSE NULL
|
||||||
|
END AS mid_price,
|
||||||
|
CASE
|
||||||
|
WHEN b.best_bid_price IS NOT NULL AND b.best_ask_price IS NOT NULL
|
||||||
|
THEN b.best_ask_price - b.best_bid_price
|
||||||
|
ELSE NULL
|
||||||
|
END AS spread_abs,
|
||||||
|
CASE
|
||||||
|
WHEN b.best_bid_price IS NOT NULL
|
||||||
|
AND b.best_ask_price IS NOT NULL
|
||||||
|
AND (b.best_bid_price + b.best_ask_price) > 0
|
||||||
|
THEN ((b.best_ask_price - b.best_bid_price) / ((b.best_bid_price + b.best_ask_price) / 2)) * 10000
|
||||||
|
ELSE NULL
|
||||||
|
END AS spread_bps,
|
||||||
|
10::integer AS depth_levels,
|
||||||
|
bids.depth_bid_base,
|
||||||
|
asks.depth_ask_base,
|
||||||
|
bids.depth_bid_usd,
|
||||||
|
asks.depth_ask_usd,
|
||||||
|
CASE
|
||||||
|
WHEN (bids.depth_bid_usd + asks.depth_ask_usd) > 0
|
||||||
|
THEN (bids.depth_bid_usd - asks.depth_ask_usd) / (bids.depth_bid_usd + asks.depth_ask_usd)
|
||||||
|
ELSE NULL
|
||||||
|
END AS imbalance,
|
||||||
|
jsonb_build_object(
|
||||||
|
'canonical_table', 'dlob_orderbook_l2_latest',
|
||||||
|
'projection', 'dlob_stats_latest_projection'
|
||||||
|
) AS raw,
|
||||||
|
b.updated_at
|
||||||
|
FROM base b
|
||||||
|
LEFT JOIN bid_levels bids USING (source, market_name)
|
||||||
|
LEFT JOIN ask_levels asks USING (source, market_name);
|
||||||
|
|||||||
@@ -31,6 +31,7 @@ patchesStrategicMerge:
|
|||||||
- patch-frontend-service.yaml
|
- patch-frontend-service.yaml
|
||||||
- patch-ingestor.yaml
|
- patch-ingestor.yaml
|
||||||
- patch-dlob-publisher.yaml
|
- patch-dlob-publisher.yaml
|
||||||
|
- patch-dlob-publisher-storage.yaml
|
||||||
- patch-dlob-server.yaml
|
- patch-dlob-server.yaml
|
||||||
- patch-dlob-worker.yaml
|
- patch-dlob-worker.yaml
|
||||||
- patch-dlob-depth-worker.yaml
|
- patch-dlob-depth-worker.yaml
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: Deployment
|
||||||
|
metadata:
|
||||||
|
name: dlob-publisher
|
||||||
|
spec:
|
||||||
|
replicas: 0
|
||||||
|
template:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- name: publisher
|
||||||
|
env:
|
||||||
|
- name: ENABLE_PERSISTENT_STORE
|
||||||
|
value: "true"
|
||||||
|
- name: DLOB_SOURCE
|
||||||
|
value: "mevnode_bot"
|
||||||
|
- name: PRICE_PRECISION
|
||||||
|
value: "1000000"
|
||||||
|
- name: BASE_PRECISION
|
||||||
|
value: "1000000000"
|
||||||
|
- name: PERSISTENT_STATS_DEPTH
|
||||||
|
value: "10"
|
||||||
|
- name: PGHOST
|
||||||
|
value: postgres
|
||||||
|
- name: PGPORT
|
||||||
|
value: "5432"
|
||||||
|
- name: PGUSER
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: trade-postgres
|
||||||
|
key: POSTGRES_USER
|
||||||
|
- name: PGPASSWORD
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: trade-postgres
|
||||||
|
key: POSTGRES_PASSWORD
|
||||||
|
- name: PGDATABASE
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: trade-postgres
|
||||||
|
key: POSTGRES_DB
|
||||||
Reference in New Issue
Block a user