diff --git a/.gitea/workflows/deploy-trade-r001-canary.yaml b/.gitea/workflows/deploy-trade-r001-canary.yaml index b9a2de3..69e6b4c 100644 --- a/.gitea/workflows/deploy-trade-r001-canary.yaml +++ b/.gitea/workflows/deploy-trade-r001-canary.yaml @@ -34,7 +34,7 @@ jobs: env: KUBECONFIG: /tmp/kubeconfig run: | - kubectl -n trade-r001-canary get secret trade-postgres trade-hasura trade-api trade-frontend-tokens trade-basic-auth trade-ingestor-tokens gitea-registry + kubectl -n trade-r001-canary get secret trade-postgres trade-hasura trade-api trade-frontend-tokens trade-basic-auth trade-ingestor-tokens trade-dlob-rpc gitea-registry - name: Recreate bootstrap jobs env: @@ -42,6 +42,13 @@ jobs: run: | kubectl -n trade-r001-canary delete job postgres-migrate hasura-bootstrap --ignore-not-found=true + - name: Apply shared host access infrastructure + env: + KUBECONFIG: /tmp/kubeconfig + run: | + kubectl apply -k environments/sol/trade-infra + kubectl -n trade-infra get svc,endpointslice + - name: Apply canary environment env: KUBECONFIG: /tmp/kubeconfig @@ -54,7 +61,14 @@ jobs: env: KUBECONFIG: /tmp/kubeconfig run: | - kubectl -n trade-r001-canary rollout restart deploy/hasura deploy/trade-api deploy/trade-frontend deploy/trade-ingestor + kubectl -n trade-r001-canary rollout restart \ + deploy/hasura \ + deploy/trade-api \ + deploy/trade-frontend \ + deploy/trade-ingestor \ + deploy/dlob-publisher-hot \ + deploy/dlob-hot-redis-to-postgres-raw-writer \ + deploy/dlob-hot-postgres-to-postgres-derived-writer - name: Wait for database and metadata bootstrap env: @@ -71,6 +85,9 @@ jobs: kubectl -n trade-r001-canary rollout status deploy/trade-api --timeout=300s kubectl -n trade-r001-canary rollout status deploy/trade-frontend --timeout=300s kubectl -n trade-r001-canary rollout status deploy/trade-ingestor --timeout=300s + kubectl -n trade-r001-canary rollout status deploy/dlob-publisher-hot --timeout=420s + kubectl -n trade-r001-canary rollout status deploy/dlob-hot-redis-to-postgres-raw-writer --timeout=300s + kubectl -n trade-r001-canary rollout status deploy/dlob-hot-postgres-to-postgres-derived-writer --timeout=300s kubectl -n trade-r001-canary get deploy,pods -o wide - name: Verify trade-ingestor runtime @@ -94,6 +111,9 @@ jobs: const targets = [ ['postgres-host.trade-infra.svc.cluster.local', 5432], ['redis-host.trade-infra.svc.cluster.local', 6379], + ['agave-rpc-host.trade-infra.svc.cluster.local', 8899], + ['agave-ws-host.trade-infra.svc.cluster.local', 8900], + ['agave-grpc-host.trade-infra.svc.cluster.local', 10000], ]; function checkSocket(host, port) { @@ -120,6 +140,63 @@ jobs: process.exit(1); }); JS + + - name: Verify DLOB hot-path runtime + env: + KUBECONFIG: /tmp/kubeconfig + run: | + kubectl -n trade-r001-canary logs deploy/dlob-publisher-hot --tail=20 + kubectl -n trade-r001-canary logs deploy/dlob-hot-redis-to-postgres-raw-writer --tail=20 + kubectl -n trade-r001-canary logs deploy/dlob-hot-postgres-to-postgres-derived-writer --tail=20 + pod_name="$(kubectl -n trade-r001-canary get pod -l app.kubernetes.io/name=trade-ingestor -o jsonpath='{.items[0].metadata.name}')" + kubectl -n trade-r001-canary exec -i "$pod_name" -- node - <<'JS' + const endpoint = 'http://hasura:8080/v1/graphql'; + const adminSecret = process.env.HASURA_ADMIN_SECRET; + const query = ` + query { + dlob_hot_derived_latest(limit: 1, order_by: { updated_at: desc }) { + market_name + updated_at + source + } + } + `; + + async function check() { + const response = await fetch(endpoint, { + method: 'POST', + headers: { + 'content-type': 'application/json', + 'x-hasura-admin-secret': adminSecret, + }, + body: JSON.stringify({ query }), + signal: AbortSignal.timeout(10000), + }); + const payload = await response.json(); + const rows = payload?.data?.dlob_hot_derived_latest || []; + if (!rows.length) { + throw new Error('No rows in dlob_hot_derived_latest yet'); + } + console.log(JSON.stringify(rows[0], null, 2)); + } + + (async () => { + for (let attempt = 0; attempt < 24; attempt += 1) { + try { + await check(); + return; + } catch (error) { + if (attempt === 23) { + throw error; + } + await new Promise((resolve) => setTimeout(resolve, 5000)); + } + } + })().catch((error) => { + console.error(String(error && error.message ? error.message : error)); + process.exit(1); + }); + JS kubectl -n trade-r001-canary exec -i "$pod_name" -- node - <<'JS' const targets = [ 'http://hasura:8080/healthz', diff --git a/environments/sol/trade-infra/README.md b/environments/sol/trade-infra/README.md new file mode 100644 index 0000000..c14a78f --- /dev/null +++ b/environments/sol/trade-infra/README.md @@ -0,0 +1,25 @@ +# trade-infra + +Shared host-backed services for the `sol` cluster. + +## Purpose + +- Expose host services into `k3s` through stable service names in namespace `trade-infra`. +- Keep host access paths reproducible in Git instead of relying on manual `kubectl` history. +- Provide cluster DNS names for: + - `Postgres` + - `Redis` + - `agave` RPC + - `agave` websocket + - `agave` Yellowstone gRPC + +## Operator Flow + +From the repository root: + +```bash +./environments/sol/trade-infra/scripts/prepare-sol-agave-access.sh +kubectl apply -k environments/sol/trade-infra +``` + +`prepare-sol-agave-access.sh` installs host-level socket proxies on `sol` so pods can reach the private validator endpoints through the host IP `149.50.96.162`. diff --git a/environments/sol/trade-infra/agave-grpc-host-endpointslice.yaml b/environments/sol/trade-infra/agave-grpc-host-endpointslice.yaml new file mode 100644 index 0000000..ef5bff1 --- /dev/null +++ b/environments/sol/trade-infra/agave-grpc-host-endpointslice.yaml @@ -0,0 +1,15 @@ +apiVersion: discovery.k8s.io/v1 +kind: EndpointSlice +metadata: + name: agave-grpc-host-sol + namespace: trade-infra + labels: + kubernetes.io/service-name: agave-grpc-host +addressType: IPv4 +ports: + - name: grpc + protocol: TCP + port: 10000 +endpoints: + - addresses: + - 149.50.96.162 diff --git a/environments/sol/trade-infra/agave-grpc-host-service.yaml b/environments/sol/trade-infra/agave-grpc-host-service.yaml new file mode 100644 index 0000000..c092b3d --- /dev/null +++ b/environments/sol/trade-infra/agave-grpc-host-service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: agave-grpc-host + namespace: trade-infra +spec: + ports: + - name: grpc + port: 10000 + targetPort: 10000 diff --git a/environments/sol/trade-infra/agave-rpc-host-endpointslice.yaml b/environments/sol/trade-infra/agave-rpc-host-endpointslice.yaml new file mode 100644 index 0000000..2521edc --- /dev/null +++ b/environments/sol/trade-infra/agave-rpc-host-endpointslice.yaml @@ -0,0 +1,15 @@ +apiVersion: discovery.k8s.io/v1 +kind: EndpointSlice +metadata: + name: agave-rpc-host-sol + namespace: trade-infra + labels: + kubernetes.io/service-name: agave-rpc-host +addressType: IPv4 +ports: + - name: rpc + protocol: TCP + port: 8899 +endpoints: + - addresses: + - 149.50.96.162 diff --git a/environments/sol/trade-infra/agave-rpc-host-service.yaml b/environments/sol/trade-infra/agave-rpc-host-service.yaml new file mode 100644 index 0000000..fd35497 --- /dev/null +++ b/environments/sol/trade-infra/agave-rpc-host-service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: agave-rpc-host + namespace: trade-infra +spec: + ports: + - name: rpc + port: 8899 + targetPort: 8899 diff --git a/environments/sol/trade-infra/agave-ws-host-endpointslice.yaml b/environments/sol/trade-infra/agave-ws-host-endpointslice.yaml new file mode 100644 index 0000000..4685397 --- /dev/null +++ b/environments/sol/trade-infra/agave-ws-host-endpointslice.yaml @@ -0,0 +1,15 @@ +apiVersion: discovery.k8s.io/v1 +kind: EndpointSlice +metadata: + name: agave-ws-host-sol + namespace: trade-infra + labels: + kubernetes.io/service-name: agave-ws-host +addressType: IPv4 +ports: + - name: ws + protocol: TCP + port: 8900 +endpoints: + - addresses: + - 149.50.96.162 diff --git a/environments/sol/trade-infra/agave-ws-host-service.yaml b/environments/sol/trade-infra/agave-ws-host-service.yaml new file mode 100644 index 0000000..9dd8153 --- /dev/null +++ b/environments/sol/trade-infra/agave-ws-host-service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: agave-ws-host + namespace: trade-infra +spec: + ports: + - name: ws + port: 8900 + targetPort: 8900 diff --git a/environments/sol/trade-infra/kustomization.yaml b/environments/sol/trade-infra/kustomization.yaml new file mode 100644 index 0000000..fd5d773 --- /dev/null +++ b/environments/sol/trade-infra/kustomization.yaml @@ -0,0 +1,17 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +namespace: trade-infra + +resources: + - namespace.yaml + - postgres-host-service.yaml + - postgres-host-endpointslice.yaml + - redis-host-service.yaml + - redis-host-endpointslice.yaml + - agave-rpc-host-service.yaml + - agave-rpc-host-endpointslice.yaml + - agave-ws-host-service.yaml + - agave-ws-host-endpointslice.yaml + - agave-grpc-host-service.yaml + - agave-grpc-host-endpointslice.yaml diff --git a/environments/sol/trade-infra/namespace.yaml b/environments/sol/trade-infra/namespace.yaml new file mode 100644 index 0000000..2830eb7 --- /dev/null +++ b/environments/sol/trade-infra/namespace.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: trade-infra diff --git a/environments/sol/trade-infra/postgres-host-endpointslice.yaml b/environments/sol/trade-infra/postgres-host-endpointslice.yaml new file mode 100644 index 0000000..cf25f04 --- /dev/null +++ b/environments/sol/trade-infra/postgres-host-endpointslice.yaml @@ -0,0 +1,15 @@ +apiVersion: discovery.k8s.io/v1 +kind: EndpointSlice +metadata: + name: postgres-host-sol + namespace: trade-infra + labels: + kubernetes.io/service-name: postgres-host +addressType: IPv4 +ports: + - name: postgres + protocol: TCP + port: 5432 +endpoints: + - addresses: + - 149.50.96.162 diff --git a/environments/sol/trade-infra/postgres-host-service.yaml b/environments/sol/trade-infra/postgres-host-service.yaml new file mode 100644 index 0000000..e7a24dd --- /dev/null +++ b/environments/sol/trade-infra/postgres-host-service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: postgres-host + namespace: trade-infra +spec: + ports: + - name: postgres + port: 5432 + targetPort: 5432 diff --git a/environments/sol/trade-infra/redis-host-endpointslice.yaml b/environments/sol/trade-infra/redis-host-endpointslice.yaml new file mode 100644 index 0000000..63b5014 --- /dev/null +++ b/environments/sol/trade-infra/redis-host-endpointslice.yaml @@ -0,0 +1,15 @@ +apiVersion: discovery.k8s.io/v1 +kind: EndpointSlice +metadata: + name: redis-host-sol + namespace: trade-infra + labels: + kubernetes.io/service-name: redis-host +addressType: IPv4 +ports: + - name: redis + protocol: TCP + port: 6379 +endpoints: + - addresses: + - 149.50.96.162 diff --git a/environments/sol/trade-infra/redis-host-service.yaml b/environments/sol/trade-infra/redis-host-service.yaml new file mode 100644 index 0000000..fb5ca43 --- /dev/null +++ b/environments/sol/trade-infra/redis-host-service.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: redis-host + namespace: trade-infra +spec: + ports: + - name: redis + port: 6379 + targetPort: 6379 diff --git a/environments/sol/trade-infra/scripts/prepare-sol-agave-access.sh b/environments/sol/trade-infra/scripts/prepare-sol-agave-access.sh new file mode 100755 index 0000000..b1175b6 --- /dev/null +++ b/environments/sol/trade-infra/scripts/prepare-sol-agave-access.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +TARGET_HOST="${TARGET_HOST:-mevnode}" +HOST_IP="${HOST_IP:-149.50.96.162}" +POD_CIDR="${POD_CIDR:-10.42.0.0/24}" + +ssh_target() { + ssh -o StrictHostKeyChecking=no "$TARGET_HOST" "$@" +} + +install_unit() { + local unit_path="$1" + ssh_target "sudo tee ${unit_path} >/dev/null" +} + +cat </dev/null" + fi +} + +ensure_ufw_rule 8899 k3s-pods-agave-rpc +ensure_ufw_rule 8900 k3s-pods-agave-ws +ensure_ufw_rule 10000 k3s-pods-agave-grpc + +ssh_target "sudo systemctl status --no-pager agave-rpc-k3s.socket agave-ws-k3s.socket agave-grpc-k3s.socket | sed -n '1,80p'" +ssh_target "sudo ss -ltnp | egrep ':(8899|8900|10000)\\b' | sed -n '1,40p'" diff --git a/environments/sol/trade-r001-canary/README.md b/environments/sol/trade-r001-canary/README.md index 20acdbc..252ae13 100644 --- a/environments/sol/trade-r001-canary/README.md +++ b/environments/sol/trade-r001-canary/README.md @@ -7,7 +7,7 @@ Minimal canary namespace for migration baseline `R001` on `sol`. - Reserve a dedicated namespace for the first reconstructed trade deployment. - Put hard upper bounds on namespace-level CPU, memory, object count, and PVC growth before application manifests land. - Verify that workloads in the namespace can resolve and reach the shared `trade-infra` services for `Postgres` and `Redis`. -- Recreate the `R001` application surface in a controlled way: `Hasura`, `trade-api`, `trade-frontend`, and the first canary `trade-ingestor` path. +- Recreate the `R001` application surface in a controlled way: `Hasura`, `trade-api`, `trade-frontend`, the first canary `trade-ingestor` path, and the first DLOB hot-path components. ## Current Guardrails @@ -34,16 +34,21 @@ Minimal canary namespace for migration baseline `R001` on `sol`. - Current shared infrastructure endpoints expected by canary workloads: - `postgres-host.trade-infra.svc.cluster.local:5432` - `redis-host.trade-infra.svc.cluster.local:6379` + - `agave-rpc-host.trade-infra.svc.cluster.local:8899` + - `agave-ws-host.trade-infra.svc.cluster.local:8900` + - `agave-grpc-host.trade-infra.svc.cluster.local:10000` ## Application Surface - `postgres` in this namespace is an `ExternalName` alias that points to `postgres-host.trade-infra.svc.cluster.local`. - `Hasura` uses the live `R001` secrets copied from `trade-staging`, but connects to the host `Postgres` on `sol`. - `trade-api` and `trade-frontend` use the current live images from Gitea registry and the same bootstrap wrapper/config pattern as the source environment. +- `dlob-publisher-hot` now targets the host validator on `sol` through `trade-infra` services and writes `dlob-hot:*` into the shared Redis host service. +- `dlob-hot-redis-to-postgres-raw-writer` and `dlob-hot-postgres-to-postgres-derived-writer` rebuild the first live DLOB derived path on `sol`. - The canary workflow re-runs: - `postgres-migrate` - `hasura-bootstrap` - before it waits for `Hasura`, `trade-api`, `trade-frontend`, and `trade-ingestor` to become healthy. + before it waits for `Hasura`, `trade-api`, `trade-frontend`, `trade-ingestor`, and the DLOB hot-path deployments to become healthy. - The current canary `trade-ingestor` is intentionally pinned to the schema already reconstructed on `sol` and reads from `dlob_stats_latest`. - The exact live `R001` ingestor path that reads `dlob_*_derived_latest` remains a follow-up substep after the DLOB writer chain is reconstructed. @@ -52,8 +57,11 @@ Minimal canary namespace for migration baseline `R001` on `sol`. From the repository root: ```bash +./environments/sol/trade-infra/scripts/prepare-sol-agave-access.sh +kubectl apply -k environments/sol/trade-infra ./environments/sol/trade-r001-canary/scripts/prepare-sol-postgres.sh ./environments/sol/trade-r001-canary/scripts/create-gitea-registry-secret.sh +./environments/sol/trade-r001-canary/scripts/create-trade-dlob-rpc-secret.sh ./environments/sol/trade-r001-canary/scripts/sync-live-secrets.sh ``` diff --git a/environments/sol/trade-r001-canary/assets/hasura/hasura-bootstrap.mjs b/environments/sol/trade-r001-canary/assets/hasura/hasura-bootstrap.mjs index 2c09cf4..8b555f1 100644 --- a/environments/sol/trade-r001-canary/assets/hasura/hasura-bootstrap.mjs +++ b/environments/sol/trade-r001-canary/assets/hasura/hasura-bootstrap.mjs @@ -98,12 +98,17 @@ async function main() { const source = 'default'; const baseTicks = { schema: 'public', name: 'drift_ticks' }; + const dlobHotSnapshotLatestTable = { schema: 'public', name: 'dlob_hot_snapshot_latest' }; + const dlobHotDerivedLatestTable = { schema: 'public', name: 'dlob_hot_derived_latest' }; + const dlobAllDerivedLatestTable = { schema: 'public', name: 'dlob_all_derived_latest' }; const dlobL2LatestTable = { schema: 'public', name: 'dlob_l2_latest' }; const dlobStatsLatestTable = { schema: 'public', name: 'dlob_stats_latest' }; const dlobDepthBpsLatestTable = { schema: 'public', name: 'dlob_depth_bps_latest' }; const dlobSlippageLatestTable = { schema: 'public', name: 'dlob_slippage_latest' }; const dlobSlippageLatestV2Table = { schema: 'public', name: 'dlob_slippage_latest_v2' }; const candlesCacheTable = { schema: 'public', name: 'drift_candles_cache' }; + const dlobHotDerivedTsTable = { schema: 'public', name: 'dlob_hot_derived_ts' }; + const dlobAllDerivedTsTable = { schema: 'public', name: 'dlob_all_derived_ts' }; const dlobStatsTsTable = { schema: 'public', name: 'dlob_stats_ts' }; const dlobDepthBpsTsTable = { schema: 'public', name: 'dlob_depth_bps_ts' }; const dlobSlippageTsTable = { schema: 'public', name: 'dlob_slippage_ts' }; @@ -373,6 +378,171 @@ async function main() { 'updated_at', ], { publicFilter: dlobPublicFilter }); + await ensurePublicSelectTable(dlobHotSnapshotLatestTable, [ + 'source', + 'redis_key', + 'snapshot_kind', + 'market_type', + 'market_index', + 'market_name', + 'is_indicative', + 'ts_ms', + 'slot', + 'market_slot', + 'payload_hash', + 'mark_price_raw', + 'oracle_price_raw', + 'best_bid_price_raw', + 'best_ask_price_raw', + 'spread_pct_raw', + 'spread_quote_raw', + 'oracle_data', + 'bids', + 'asks', + 'payload', + 'updated_at', + ]); + + await ensurePublicSelectTable(dlobHotDerivedLatestTable, [ + 'source', + 'market_type', + 'market_index', + 'market_name', + 'is_indicative', + 'ts_ms', + 'slot', + 'market_slot', + 'mark_price', + 'oracle_price', + 'best_bid_price', + 'best_ask_price', + 'mid_price', + 'spread_quote', + 'spread_bps', + 'depth_levels', + 'bid_levels', + 'ask_levels', + 'top_bid_size', + 'top_ask_size', + 'top_bid_notional', + 'top_ask_notional', + 'depth_bid_base', + 'depth_ask_base', + 'depth_bid_quote', + 'depth_ask_quote', + 'imbalance', + 'bids_norm', + 'asks_norm', + 'raw_payload_hash', + 'updated_at', + ]); + + await ensurePublicSelectTable(dlobAllDerivedLatestTable, [ + 'source', + 'market_type', + 'market_index', + 'market_name', + 'is_indicative', + 'ts_ms', + 'slot', + 'market_slot', + 'mark_price', + 'oracle_price', + 'best_bid_price', + 'best_ask_price', + 'mid_price', + 'spread_quote', + 'spread_bps', + 'depth_levels', + 'bid_levels', + 'ask_levels', + 'top_bid_size', + 'top_ask_size', + 'top_bid_notional', + 'top_ask_notional', + 'depth_bid_base', + 'depth_ask_base', + 'depth_bid_quote', + 'depth_ask_quote', + 'imbalance', + 'bids_norm', + 'asks_norm', + 'raw_payload_hash', + 'updated_at', + ]); + + await ensurePublicSelectTable(dlobHotDerivedTsTable, [ + 'event_ts', + 'id', + 'source', + 'market_type', + 'market_index', + 'market_name', + 'is_indicative', + 'ts_ms', + 'slot', + 'market_slot', + 'mark_price', + 'oracle_price', + 'best_bid_price', + 'best_ask_price', + 'mid_price', + 'spread_quote', + 'spread_bps', + 'depth_levels', + 'bid_levels', + 'ask_levels', + 'top_bid_size', + 'top_ask_size', + 'top_bid_notional', + 'top_ask_notional', + 'depth_bid_base', + 'depth_ask_base', + 'depth_bid_quote', + 'depth_ask_quote', + 'imbalance', + 'bids_norm', + 'asks_norm', + 'raw_payload_hash', + 'inserted_at', + ]); + + await ensurePublicSelectTable(dlobAllDerivedTsTable, [ + 'event_ts', + 'id', + 'source', + 'market_type', + 'market_index', + 'market_name', + 'is_indicative', + 'ts_ms', + 'slot', + 'market_slot', + 'mark_price', + 'oracle_price', + 'best_bid_price', + 'best_ask_price', + 'mid_price', + 'spread_quote', + 'spread_bps', + 'depth_levels', + 'bid_levels', + 'ask_levels', + 'top_bid_size', + 'top_ask_size', + 'top_bid_notional', + 'top_ask_notional', + 'depth_bid_base', + 'depth_ask_base', + 'depth_bid_quote', + 'depth_ask_quote', + 'imbalance', + 'bids_norm', + 'asks_norm', + 'raw_payload_hash', + 'inserted_at', + ]); + await ensurePublicSelectTable(dlobStatsTsTable, [ 'ts', 'id', diff --git a/environments/sol/trade-r001-canary/assets/postgres/001_init.sql b/environments/sol/trade-r001-canary/assets/postgres/001_init.sql index d7a2b32..6feeece 100644 --- a/environments/sol/trade-r001-canary/assets/postgres/001_init.sql +++ b/environments/sol/trade-r001-canary/assets/postgres/001_init.sql @@ -858,3 +858,271 @@ BEGIN PERFORM add_retention_policy('dlob_slippage_ts_v2', INTERVAL '7 days'); EXCEPTION WHEN OTHERS THEN END $$; + +-- Canonical raw DLOB hot snapshots written from Redis to PostgreSQL. +CREATE TABLE IF NOT EXISTS public.dlob_hot_snapshot_latest ( + source TEXT NOT NULL, + redis_key TEXT NOT NULL, + snapshot_kind TEXT NOT NULL CHECK (snapshot_kind IN ('orderbook_l2', 'orderbook_l3', 'best_makers')), + market_type TEXT NOT NULL, + market_index INTEGER NOT NULL, + market_name TEXT NOT NULL, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts_ms BIGINT NOT NULL, + slot BIGINT, + market_slot BIGINT, + payload_hash TEXT NOT NULL, + mark_price_raw TEXT, + oracle_price_raw TEXT, + best_bid_price_raw TEXT, + best_ask_price_raw TEXT, + spread_pct_raw TEXT, + spread_quote_raw TEXT, + oracle_data JSONB, + bids JSONB, + asks JSONB, + payload JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (source, market_type, market_index, snapshot_kind, is_indicative) +); + +CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_latest_updated_at_idx + ON public.dlob_hot_snapshot_latest (updated_at DESC); + +CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_latest_source_market_idx + ON public.dlob_hot_snapshot_latest (source, market_type, market_index, updated_at DESC); + +CREATE TABLE IF NOT EXISTS public.dlob_hot_snapshot_ts ( + event_ts TIMESTAMPTZ NOT NULL, + id BIGSERIAL NOT NULL, + source TEXT NOT NULL, + redis_key TEXT NOT NULL, + snapshot_kind TEXT NOT NULL CHECK (snapshot_kind IN ('orderbook_l2', 'orderbook_l3', 'best_makers')), + market_type TEXT NOT NULL, + market_index INTEGER NOT NULL, + market_name TEXT NOT NULL, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts_ms BIGINT NOT NULL, + slot BIGINT, + market_slot BIGINT, + payload_hash TEXT NOT NULL, + mark_price_raw TEXT, + oracle_price_raw TEXT, + best_bid_price_raw TEXT, + best_ask_price_raw TEXT, + spread_pct_raw TEXT, + spread_quote_raw TEXT, + oracle_data JSONB, + bids JSONB, + asks JSONB, + payload JSONB NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (event_ts, id) +); + +SELECT create_hypertable('dlob_hot_snapshot_ts', 'event_ts', if_not_exists => TRUE, migrate_data => TRUE); + +CREATE UNIQUE INDEX IF NOT EXISTS dlob_hot_snapshot_ts_dedupe_idx + ON public.dlob_hot_snapshot_ts (event_ts, source, redis_key, payload_hash); + +CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_ts_market_ts_desc_idx + ON public.dlob_hot_snapshot_ts (market_type, market_index, event_ts DESC); + +CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_ts_source_market_ts_desc_idx + ON public.dlob_hot_snapshot_ts (source, market_type, market_index, event_ts DESC); + +CREATE TABLE IF NOT EXISTS public.dlob_hot_derived_latest ( + source TEXT NOT NULL, + market_type TEXT NOT NULL, + market_index INTEGER NOT NULL, + market_name TEXT NOT NULL, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts_ms BIGINT NOT NULL, + slot BIGINT, + market_slot BIGINT, + mark_price NUMERIC, + oracle_price NUMERIC, + best_bid_price NUMERIC, + best_ask_price NUMERIC, + mid_price NUMERIC, + spread_quote NUMERIC, + spread_bps NUMERIC, + depth_levels INTEGER NOT NULL, + bid_levels INTEGER NOT NULL, + ask_levels INTEGER NOT NULL, + top_bid_size NUMERIC, + top_ask_size NUMERIC, + top_bid_notional NUMERIC, + top_ask_notional NUMERIC, + depth_bid_base NUMERIC NOT NULL DEFAULT 0, + depth_ask_base NUMERIC NOT NULL DEFAULT 0, + depth_bid_quote NUMERIC NOT NULL DEFAULT 0, + depth_ask_quote NUMERIC NOT NULL DEFAULT 0, + imbalance NUMERIC, + bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + raw_payload_hash TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (source, market_type, market_index, is_indicative) +); + +CREATE INDEX IF NOT EXISTS dlob_hot_derived_latest_updated_at_idx + ON public.dlob_hot_derived_latest (updated_at DESC); + +CREATE INDEX IF NOT EXISTS dlob_hot_derived_latest_source_market_idx + ON public.dlob_hot_derived_latest (source, market_type, market_index, updated_at DESC); + +CREATE TABLE IF NOT EXISTS public.dlob_hot_derived_ts ( + event_ts TIMESTAMPTZ NOT NULL, + id BIGSERIAL NOT NULL, + source TEXT NOT NULL, + market_type TEXT NOT NULL, + market_index INTEGER NOT NULL, + market_name TEXT NOT NULL, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts_ms BIGINT NOT NULL, + slot BIGINT, + market_slot BIGINT, + mark_price NUMERIC, + oracle_price NUMERIC, + best_bid_price NUMERIC, + best_ask_price NUMERIC, + mid_price NUMERIC, + spread_quote NUMERIC, + spread_bps NUMERIC, + depth_levels INTEGER NOT NULL, + bid_levels INTEGER NOT NULL, + ask_levels INTEGER NOT NULL, + top_bid_size NUMERIC, + top_ask_size NUMERIC, + top_bid_notional NUMERIC, + top_ask_notional NUMERIC, + depth_bid_base NUMERIC NOT NULL DEFAULT 0, + depth_ask_base NUMERIC NOT NULL DEFAULT 0, + depth_bid_quote NUMERIC NOT NULL DEFAULT 0, + depth_ask_quote NUMERIC NOT NULL DEFAULT 0, + imbalance NUMERIC, + bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + raw_payload_hash TEXT NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (event_ts, id) +); + +SELECT create_hypertable('dlob_hot_derived_ts', 'event_ts', if_not_exists => TRUE, migrate_data => TRUE); + +CREATE UNIQUE INDEX IF NOT EXISTS dlob_hot_derived_ts_dedupe_idx + ON public.dlob_hot_derived_ts (event_ts, source, market_type, market_index, is_indicative, raw_payload_hash); + +CREATE INDEX IF NOT EXISTS dlob_hot_derived_ts_market_ts_desc_idx + ON public.dlob_hot_derived_ts (market_type, market_index, event_ts DESC); + +CREATE INDEX IF NOT EXISTS dlob_hot_derived_ts_source_market_ts_desc_idx + ON public.dlob_hot_derived_ts (source, market_type, market_index, event_ts DESC); + +DO $$ +BEGIN + PERFORM add_retention_policy('dlob_hot_snapshot_ts', INTERVAL '30 days'); +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; + +DO $$ +BEGIN + PERFORM add_retention_policy('dlob_hot_derived_ts', INTERVAL '180 days'); +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; + +CREATE TABLE IF NOT EXISTS public.dlob_all_derived_latest ( + source TEXT NOT NULL, + market_type TEXT NOT NULL, + market_index INTEGER NOT NULL, + market_name TEXT NOT NULL, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts_ms BIGINT NOT NULL, + slot BIGINT, + market_slot BIGINT, + mark_price NUMERIC, + oracle_price NUMERIC, + best_bid_price NUMERIC, + best_ask_price NUMERIC, + mid_price NUMERIC, + spread_quote NUMERIC, + spread_bps NUMERIC, + depth_levels INTEGER NOT NULL, + bid_levels INTEGER NOT NULL, + ask_levels INTEGER NOT NULL, + top_bid_size NUMERIC, + top_ask_size NUMERIC, + top_bid_notional NUMERIC, + top_ask_notional NUMERIC, + depth_bid_base NUMERIC NOT NULL DEFAULT 0, + depth_ask_base NUMERIC NOT NULL DEFAULT 0, + depth_bid_quote NUMERIC NOT NULL DEFAULT 0, + depth_ask_quote NUMERIC NOT NULL DEFAULT 0, + imbalance NUMERIC, + bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + raw_payload_hash TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (source, market_type, market_index, is_indicative) +); + +CREATE INDEX IF NOT EXISTS dlob_all_derived_latest_updated_at_idx + ON public.dlob_all_derived_latest (updated_at DESC); + +CREATE INDEX IF NOT EXISTS dlob_all_derived_latest_source_market_idx + ON public.dlob_all_derived_latest (source, market_type, market_index, updated_at DESC); + +CREATE TABLE IF NOT EXISTS public.dlob_all_derived_ts ( + event_ts TIMESTAMPTZ NOT NULL, + id BIGSERIAL NOT NULL, + source TEXT NOT NULL, + market_type TEXT NOT NULL, + market_index INTEGER NOT NULL, + market_name TEXT NOT NULL, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts_ms BIGINT NOT NULL, + slot BIGINT, + market_slot BIGINT, + mark_price NUMERIC, + oracle_price NUMERIC, + best_bid_price NUMERIC, + best_ask_price NUMERIC, + mid_price NUMERIC, + spread_quote NUMERIC, + spread_bps NUMERIC, + depth_levels INTEGER NOT NULL, + bid_levels INTEGER NOT NULL, + ask_levels INTEGER NOT NULL, + top_bid_size NUMERIC, + top_ask_size NUMERIC, + top_bid_notional NUMERIC, + top_ask_notional NUMERIC, + depth_bid_base NUMERIC NOT NULL DEFAULT 0, + depth_ask_base NUMERIC NOT NULL DEFAULT 0, + depth_bid_quote NUMERIC NOT NULL DEFAULT 0, + depth_ask_quote NUMERIC NOT NULL DEFAULT 0, + imbalance NUMERIC, + bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + raw_payload_hash TEXT NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (event_ts, id) +); + +SELECT create_hypertable('dlob_all_derived_ts', 'event_ts', if_not_exists => TRUE, migrate_data => TRUE); + +CREATE UNIQUE INDEX IF NOT EXISTS dlob_all_derived_ts_dedupe_idx + ON public.dlob_all_derived_ts (event_ts, source, market_type, market_index, is_indicative, raw_payload_hash); + +CREATE INDEX IF NOT EXISTS dlob_all_derived_ts_market_ts_desc_idx + ON public.dlob_all_derived_ts (market_type, market_index, event_ts DESC); + +CREATE INDEX IF NOT EXISTS dlob_all_derived_ts_source_market_ts_desc_idx + ON public.dlob_all_derived_ts (source, market_type, market_index, event_ts DESC); + +DO $$ +BEGIN + PERFORM add_retention_policy('dlob_all_derived_ts', INTERVAL '30 days'); +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; diff --git a/environments/sol/trade-r001-canary/dlob-hot-postgres-to-postgres-derived-writer-deployment.yaml b/environments/sol/trade-r001-canary/dlob-hot-postgres-to-postgres-derived-writer-deployment.yaml new file mode 100644 index 0000000..1a88d80 --- /dev/null +++ b/environments/sol/trade-r001-canary/dlob-hot-postgres-to-postgres-derived-writer-deployment.yaml @@ -0,0 +1,63 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dlob-hot-postgres-to-postgres-derived-writer + namespace: trade-r001-canary +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: dlob-hot-postgres-to-postgres-derived-writer + template: + metadata: + labels: + app.kubernetes.io/name: dlob-hot-postgres-to-postgres-derived-writer + spec: + imagePullSecrets: + - name: gitea-registry + containers: + - name: writer + image: gitea.mpabi.pl/trade/trade-dlob-server:hot-pg-events-20260320-205517 + imagePullPolicy: IfNotPresent + command: + - node + - /lib/scripts/dlobHotPostgresToPostgresDerivedWriter.js + env: + - name: DLOB_SOURCE + value: mevnode_bot_hot_derived + - name: RAW_SOURCE + value: mevnode_bot_hot_raw + - name: DLOB_POLL_MS + value: "5000" + - name: NORMALIZED_DEPTH + value: "10" + - name: PRICE_PRECISION + value: "1000000" + - name: BASE_PRECISION + value: "1000000000" + - name: PGHOST + value: postgres + - name: PGPORT + value: "5432" + - name: PGUSER + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_PASSWORD + - name: PGDATABASE + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_DB + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi diff --git a/environments/sol/trade-r001-canary/dlob-hot-redis-to-postgres-raw-writer-deployment.yaml b/environments/sol/trade-r001-canary/dlob-hot-redis-to-postgres-raw-writer-deployment.yaml new file mode 100644 index 0000000..bc63889 --- /dev/null +++ b/environments/sol/trade-r001-canary/dlob-hot-redis-to-postgres-raw-writer-deployment.yaml @@ -0,0 +1,61 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dlob-hot-redis-to-postgres-raw-writer + namespace: trade-r001-canary +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: dlob-hot-redis-to-postgres-raw-writer + template: + metadata: + labels: + app.kubernetes.io/name: dlob-hot-redis-to-postgres-raw-writer + spec: + imagePullSecrets: + - name: gitea-registry + containers: + - name: writer + image: gitea.mpabi.pl/trade/trade-dlob-server:hot-pg-events-20260320-205517 + imagePullPolicy: IfNotPresent + command: + - node + - /lib/scripts/dlobHotRedisToPostgresRawWriter.js + env: + - name: DLOB_SOURCE + value: mevnode_bot_hot_raw + - name: REDIS_HOST + value: dlob-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_KEY_PREFIX + value: "dlob-hot:" + - name: DLOB_POLL_MS + value: "5000" + - name: PGHOST + value: postgres + - name: PGPORT + value: "5432" + - name: PGUSER + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_PASSWORD + - name: PGDATABASE + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_DB + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi diff --git a/environments/sol/trade-r001-canary/dlob-publisher-hot-deployment.yaml b/environments/sol/trade-r001-canary/dlob-publisher-hot-deployment.yaml new file mode 100644 index 0000000..b3cb487 --- /dev/null +++ b/environments/sol/trade-r001-canary/dlob-publisher-hot-deployment.yaml @@ -0,0 +1,130 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dlob-publisher-hot + namespace: trade-r001-canary +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: dlob-publisher-hot + template: + metadata: + labels: + app.kubernetes.io/name: dlob-publisher-hot + spec: + imagePullSecrets: + - name: gitea-registry + containers: + - name: publisher + image: gitea.mpabi.pl/trade/trade-dlob-server:grpc-teardown-fix-20260402-113736 + imagePullPolicy: IfNotPresent + command: + - node + - /lib/publishers/dlobPublisher.js + ports: + - name: http + containerPort: 8080 + env: + - name: FETCH_CONNECT_TIMEOUT_MS + value: "15000" + - name: FETCH_HEADERS_TIMEOUT_MS + value: "300000" + - name: FETCH_BODY_TIMEOUT_MS + value: "300000" + - name: ENABLE_PERSISTENT_STORE + value: "true" + - name: DLOB_SOURCE + value: mevnode_bot_hot + - name: PRICE_PRECISION + value: "1000000" + - name: BASE_PRECISION + value: "1000000000" + - name: PERSISTENT_STATS_DEPTH + value: "10" + - name: PGHOST + value: postgres + - name: PGPORT + value: "5432" + - name: PGUSER + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_PASSWORD + - name: PGDATABASE + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_DB + - name: PERP_MARKETS_TO_LOAD + value: "0,20,72" + - name: USE_GRPC + value: "true" + - name: USE_WEBSOCKET + value: "true" + - name: DISABLE_GPA_REFRESH + value: "true" + - name: GRPC_CLIENT + value: yellowstone + - name: GRPC_ENDPOINT + valueFrom: + secretKeyRef: + name: trade-dlob-rpc + key: GRPC_ENDPOINT + - name: TOKEN + valueFrom: + secretKeyRef: + name: trade-dlob-rpc + key: TOKEN + - name: RUNNING_LOCAL + value: "true" + - name: LOCAL_CACHE + value: "true" + - name: ENV + value: mainnet-beta + - name: USE_ORDER_SUBSCRIBER + value: "true" + - name: ELASTICACHE_HOST + value: dlob-redis + - name: ELASTICACHE_PORT + value: "6379" + - name: REDIS_CLIENT + value: DLOB_HOT + - name: ENDPOINT + valueFrom: + secretKeyRef: + name: trade-dlob-rpc + key: ENDPOINT + - name: WS_ENDPOINT + valueFrom: + secretKeyRef: + name: trade-dlob-rpc + key: WS_ENDPOINT + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: "1" + memory: 1Gi + readinessProbe: + httpGet: + path: /startup + port: http + initialDelaySeconds: 120 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 30 + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 240 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 10 diff --git a/environments/sol/trade-r001-canary/dlob-redis-alias-service.yaml b/environments/sol/trade-r001-canary/dlob-redis-alias-service.yaml new file mode 100644 index 0000000..4352606 --- /dev/null +++ b/environments/sol/trade-r001-canary/dlob-redis-alias-service.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Service +metadata: + name: dlob-redis + namespace: trade-r001-canary +spec: + type: ExternalName + externalName: redis-host.trade-infra.svc.cluster.local + ports: + - name: redis + port: 6379 + targetPort: 6379 diff --git a/environments/sol/trade-r001-canary/hasura-bootstrap-job.yaml b/environments/sol/trade-r001-canary/hasura-bootstrap-job.yaml index 00aae01..ccddd85 100644 --- a/environments/sol/trade-r001-canary/hasura-bootstrap-job.yaml +++ b/environments/sol/trade-r001-canary/hasura-bootstrap-job.yaml @@ -15,6 +15,13 @@ spec: envFrom: - secretRef: name: trade-postgres + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi command: - sh - -ec @@ -38,6 +45,13 @@ spec: value: drift_ticks - name: CANDLES_FUNCTION value: get_drift_candles + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi command: - node - /app/hasura-bootstrap.mjs diff --git a/environments/sol/trade-r001-canary/kustomization.yaml b/environments/sol/trade-r001-canary/kustomization.yaml index ca36f42..bdaab46 100644 --- a/environments/sol/trade-r001-canary/kustomization.yaml +++ b/environments/sol/trade-r001-canary/kustomization.yaml @@ -11,6 +11,7 @@ resources: - resourcequota.yaml - limitrange.yaml - postgres-alias-service.yaml + - dlob-redis-alias-service.yaml - hasura-service.yaml - hasura-deployment.yaml - postgres-migrate-job.yaml @@ -20,6 +21,9 @@ resources: - trade-frontend-service.yaml - trade-frontend-deployment.yaml - trade-ingestor-deployment.yaml + - dlob-publisher-hot-deployment.yaml + - dlob-hot-redis-to-postgres-raw-writer-deployment.yaml + - dlob-hot-postgres-to-postgres-derived-writer-deployment.yaml configMapGenerator: - name: postgres-initdb diff --git a/environments/sol/trade-r001-canary/postgres-migrate-job.yaml b/environments/sol/trade-r001-canary/postgres-migrate-job.yaml index 25124ba..6bae4b8 100644 --- a/environments/sol/trade-r001-canary/postgres-migrate-job.yaml +++ b/environments/sol/trade-r001-canary/postgres-migrate-job.yaml @@ -15,6 +15,13 @@ spec: envFrom: - secretRef: name: trade-postgres + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: 250m + memory: 256Mi command: - sh - -ec diff --git a/environments/sol/trade-r001-canary/scripts/create-trade-dlob-rpc-secret.sh b/environments/sol/trade-r001-canary/scripts/create-trade-dlob-rpc-secret.sh new file mode 100755 index 0000000..0425bf7 --- /dev/null +++ b/environments/sol/trade-r001-canary/scripts/create-trade-dlob-rpc-secret.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -euo pipefail + +TARGET_HOST="${TARGET_HOST:-mevnode}" +TARGET_NAMESPACE="${TARGET_NAMESPACE:-trade-r001-canary}" + +RPC_URL="${RPC_URL:-http://agave-rpc-host.trade-infra.svc.cluster.local:8899}" +WS_URL="${WS_URL:-ws://agave-ws-host.trade-infra.svc.cluster.local:8900}" +GRPC_URL="${GRPC_URL:-http://agave-grpc-host.trade-infra.svc.cluster.local:10000}" + +ssh_target() { + ssh -o StrictHostKeyChecking=no "$TARGET_HOST" "$@" +} + +TOKEN="$(ssh_target 'sudo cat /etc/agave/geyser.x_token')" + +ssh_target "sudo k3s kubectl get ns ${TARGET_NAMESPACE} >/dev/null 2>&1 || sudo k3s kubectl create ns ${TARGET_NAMESPACE} >/dev/null" +ssh_target "sudo k3s kubectl -n ${TARGET_NAMESPACE} create secret generic trade-dlob-rpc \ + --from-literal=ENDPOINT='${RPC_URL}' \ + --from-literal=WS_ENDPOINT='${WS_URL}' \ + --from-literal=GRPC_ENDPOINT='${GRPC_URL}' \ + --from-literal=TOKEN='${TOKEN}' \ + --dry-run=client -o yaml | sudo k3s kubectl apply -f - >/dev/null" + +echo "Created trade-dlob-rpc in ${TARGET_NAMESPACE}"