feat(sol): add agave-backed dlob hot path for canary
Some checks failed
deploy-trade-r001-canary / apply (push) Failing after 5m41s

This commit is contained in:
mpabi
2026-04-12 18:10:42 +02:00
parent 948c37c3f5
commit e1e993e2ac
26 changed files with 1110 additions and 4 deletions

View File

@@ -34,7 +34,7 @@ jobs:
env:
KUBECONFIG: /tmp/kubeconfig
run: |
kubectl -n trade-r001-canary get secret trade-postgres trade-hasura trade-api trade-frontend-tokens trade-basic-auth trade-ingestor-tokens gitea-registry
kubectl -n trade-r001-canary get secret trade-postgres trade-hasura trade-api trade-frontend-tokens trade-basic-auth trade-ingestor-tokens trade-dlob-rpc gitea-registry
- name: Recreate bootstrap jobs
env:
@@ -42,6 +42,13 @@ jobs:
run: |
kubectl -n trade-r001-canary delete job postgres-migrate hasura-bootstrap --ignore-not-found=true
- name: Apply shared host access infrastructure
env:
KUBECONFIG: /tmp/kubeconfig
run: |
kubectl apply -k environments/sol/trade-infra
kubectl -n trade-infra get svc,endpointslice
- name: Apply canary environment
env:
KUBECONFIG: /tmp/kubeconfig
@@ -54,7 +61,14 @@ jobs:
env:
KUBECONFIG: /tmp/kubeconfig
run: |
kubectl -n trade-r001-canary rollout restart deploy/hasura deploy/trade-api deploy/trade-frontend deploy/trade-ingestor
kubectl -n trade-r001-canary rollout restart \
deploy/hasura \
deploy/trade-api \
deploy/trade-frontend \
deploy/trade-ingestor \
deploy/dlob-publisher-hot \
deploy/dlob-hot-redis-to-postgres-raw-writer \
deploy/dlob-hot-postgres-to-postgres-derived-writer
- name: Wait for database and metadata bootstrap
env:
@@ -71,6 +85,9 @@ jobs:
kubectl -n trade-r001-canary rollout status deploy/trade-api --timeout=300s
kubectl -n trade-r001-canary rollout status deploy/trade-frontend --timeout=300s
kubectl -n trade-r001-canary rollout status deploy/trade-ingestor --timeout=300s
kubectl -n trade-r001-canary rollout status deploy/dlob-publisher-hot --timeout=420s
kubectl -n trade-r001-canary rollout status deploy/dlob-hot-redis-to-postgres-raw-writer --timeout=300s
kubectl -n trade-r001-canary rollout status deploy/dlob-hot-postgres-to-postgres-derived-writer --timeout=300s
kubectl -n trade-r001-canary get deploy,pods -o wide
- name: Verify trade-ingestor runtime
@@ -94,6 +111,9 @@ jobs:
const targets = [
['postgres-host.trade-infra.svc.cluster.local', 5432],
['redis-host.trade-infra.svc.cluster.local', 6379],
['agave-rpc-host.trade-infra.svc.cluster.local', 8899],
['agave-ws-host.trade-infra.svc.cluster.local', 8900],
['agave-grpc-host.trade-infra.svc.cluster.local', 10000],
];
function checkSocket(host, port) {
@@ -120,6 +140,63 @@ jobs:
process.exit(1);
});
JS
- name: Verify DLOB hot-path runtime
env:
KUBECONFIG: /tmp/kubeconfig
run: |
kubectl -n trade-r001-canary logs deploy/dlob-publisher-hot --tail=20
kubectl -n trade-r001-canary logs deploy/dlob-hot-redis-to-postgres-raw-writer --tail=20
kubectl -n trade-r001-canary logs deploy/dlob-hot-postgres-to-postgres-derived-writer --tail=20
pod_name="$(kubectl -n trade-r001-canary get pod -l app.kubernetes.io/name=trade-ingestor -o jsonpath='{.items[0].metadata.name}')"
kubectl -n trade-r001-canary exec -i "$pod_name" -- node - <<'JS'
const endpoint = 'http://hasura:8080/v1/graphql';
const adminSecret = process.env.HASURA_ADMIN_SECRET;
const query = `
query {
dlob_hot_derived_latest(limit: 1, order_by: { updated_at: desc }) {
market_name
updated_at
source
}
}
`;
async function check() {
const response = await fetch(endpoint, {
method: 'POST',
headers: {
'content-type': 'application/json',
'x-hasura-admin-secret': adminSecret,
},
body: JSON.stringify({ query }),
signal: AbortSignal.timeout(10000),
});
const payload = await response.json();
const rows = payload?.data?.dlob_hot_derived_latest || [];
if (!rows.length) {
throw new Error('No rows in dlob_hot_derived_latest yet');
}
console.log(JSON.stringify(rows[0], null, 2));
}
(async () => {
for (let attempt = 0; attempt < 24; attempt += 1) {
try {
await check();
return;
} catch (error) {
if (attempt === 23) {
throw error;
}
await new Promise((resolve) => setTimeout(resolve, 5000));
}
}
})().catch((error) => {
console.error(String(error && error.message ? error.message : error));
process.exit(1);
});
JS
kubectl -n trade-r001-canary exec -i "$pod_name" -- node - <<'JS'
const targets = [
'http://hasura:8080/healthz',

View File

@@ -0,0 +1,25 @@
# trade-infra
Shared host-backed services for the `sol` cluster.
## Purpose
- Expose host services into `k3s` through stable service names in namespace `trade-infra`.
- Keep host access paths reproducible in Git instead of relying on manual `kubectl` history.
- Provide cluster DNS names for:
- `Postgres`
- `Redis`
- `agave` RPC
- `agave` websocket
- `agave` Yellowstone gRPC
## Operator Flow
From the repository root:
```bash
./environments/sol/trade-infra/scripts/prepare-sol-agave-access.sh
kubectl apply -k environments/sol/trade-infra
```
`prepare-sol-agave-access.sh` installs host-level socket proxies on `sol` so pods can reach the private validator endpoints through the host IP `149.50.96.162`.

View File

@@ -0,0 +1,15 @@
apiVersion: discovery.k8s.io/v1
kind: EndpointSlice
metadata:
name: agave-grpc-host-sol
namespace: trade-infra
labels:
kubernetes.io/service-name: agave-grpc-host
addressType: IPv4
ports:
- name: grpc
protocol: TCP
port: 10000
endpoints:
- addresses:
- 149.50.96.162

View File

@@ -0,0 +1,10 @@
apiVersion: v1
kind: Service
metadata:
name: agave-grpc-host
namespace: trade-infra
spec:
ports:
- name: grpc
port: 10000
targetPort: 10000

View File

@@ -0,0 +1,15 @@
apiVersion: discovery.k8s.io/v1
kind: EndpointSlice
metadata:
name: agave-rpc-host-sol
namespace: trade-infra
labels:
kubernetes.io/service-name: agave-rpc-host
addressType: IPv4
ports:
- name: rpc
protocol: TCP
port: 8899
endpoints:
- addresses:
- 149.50.96.162

View File

@@ -0,0 +1,10 @@
apiVersion: v1
kind: Service
metadata:
name: agave-rpc-host
namespace: trade-infra
spec:
ports:
- name: rpc
port: 8899
targetPort: 8899

View File

@@ -0,0 +1,15 @@
apiVersion: discovery.k8s.io/v1
kind: EndpointSlice
metadata:
name: agave-ws-host-sol
namespace: trade-infra
labels:
kubernetes.io/service-name: agave-ws-host
addressType: IPv4
ports:
- name: ws
protocol: TCP
port: 8900
endpoints:
- addresses:
- 149.50.96.162

View File

@@ -0,0 +1,10 @@
apiVersion: v1
kind: Service
metadata:
name: agave-ws-host
namespace: trade-infra
spec:
ports:
- name: ws
port: 8900
targetPort: 8900

View File

@@ -0,0 +1,17 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
namespace: trade-infra
resources:
- namespace.yaml
- postgres-host-service.yaml
- postgres-host-endpointslice.yaml
- redis-host-service.yaml
- redis-host-endpointslice.yaml
- agave-rpc-host-service.yaml
- agave-rpc-host-endpointslice.yaml
- agave-ws-host-service.yaml
- agave-ws-host-endpointslice.yaml
- agave-grpc-host-service.yaml
- agave-grpc-host-endpointslice.yaml

View File

@@ -0,0 +1,4 @@
apiVersion: v1
kind: Namespace
metadata:
name: trade-infra

View File

@@ -0,0 +1,15 @@
apiVersion: discovery.k8s.io/v1
kind: EndpointSlice
metadata:
name: postgres-host-sol
namespace: trade-infra
labels:
kubernetes.io/service-name: postgres-host
addressType: IPv4
ports:
- name: postgres
protocol: TCP
port: 5432
endpoints:
- addresses:
- 149.50.96.162

View File

@@ -0,0 +1,10 @@
apiVersion: v1
kind: Service
metadata:
name: postgres-host
namespace: trade-infra
spec:
ports:
- name: postgres
port: 5432
targetPort: 5432

View File

@@ -0,0 +1,15 @@
apiVersion: discovery.k8s.io/v1
kind: EndpointSlice
metadata:
name: redis-host-sol
namespace: trade-infra
labels:
kubernetes.io/service-name: redis-host
addressType: IPv4
ports:
- name: redis
protocol: TCP
port: 6379
endpoints:
- addresses:
- 149.50.96.162

View File

@@ -0,0 +1,10 @@
apiVersion: v1
kind: Service
metadata:
name: redis-host
namespace: trade-infra
spec:
ports:
- name: redis
port: 6379
targetPort: 6379

View File

@@ -0,0 +1,96 @@
#!/usr/bin/env bash
set -euo pipefail
TARGET_HOST="${TARGET_HOST:-mevnode}"
HOST_IP="${HOST_IP:-149.50.96.162}"
POD_CIDR="${POD_CIDR:-10.42.0.0/24}"
ssh_target() {
ssh -o StrictHostKeyChecking=no "$TARGET_HOST" "$@"
}
install_unit() {
local unit_path="$1"
ssh_target "sudo tee ${unit_path} >/dev/null"
}
cat <<EOF | install_unit /etc/systemd/system/agave-rpc-k3s.socket
[Unit]
Description=Expose Agave RPC on host IP for k3s pods
[Socket]
ListenStream=${HOST_IP}:8899
NoDelay=true
[Install]
WantedBy=sockets.target
EOF
cat <<'EOF' | install_unit /etc/systemd/system/agave-rpc-k3s.service
[Unit]
Description=Proxy Agave RPC from host IP to localhost
[Service]
ExecStart=/lib/systemd/systemd-socket-proxyd 127.0.0.1:8899
PrivateNetwork=no
EOF
cat <<EOF | install_unit /etc/systemd/system/agave-ws-k3s.socket
[Unit]
Description=Expose Agave websocket on host IP for k3s pods
[Socket]
ListenStream=${HOST_IP}:8900
NoDelay=true
[Install]
WantedBy=sockets.target
EOF
cat <<'EOF' | install_unit /etc/systemd/system/agave-ws-k3s.service
[Unit]
Description=Proxy Agave websocket from host IP to localhost
[Service]
ExecStart=/lib/systemd/systemd-socket-proxyd 127.0.0.1:8900
PrivateNetwork=no
EOF
cat <<EOF | install_unit /etc/systemd/system/agave-grpc-k3s.socket
[Unit]
Description=Expose Agave Yellowstone gRPC on host IP for k3s pods
[Socket]
ListenStream=${HOST_IP}:10000
NoDelay=true
[Install]
WantedBy=sockets.target
EOF
cat <<'EOF' | install_unit /etc/systemd/system/agave-grpc-k3s.service
[Unit]
Description=Proxy Agave Yellowstone gRPC from host IP to WireGuard IP
[Service]
ExecStart=/lib/systemd/systemd-socket-proxyd 10.91.0.1:10000
PrivateNetwork=no
EOF
ssh_target "sudo systemctl daemon-reload"
ssh_target "sudo systemctl enable --now agave-rpc-k3s.socket agave-ws-k3s.socket agave-grpc-k3s.socket"
ensure_ufw_rule() {
local port="$1"
local comment="$2"
if ! ssh_target "sudo ufw status numbered | grep -Fq '${port}/tcp on cni0'"; then
ssh_target "sudo ufw allow in on cni0 from ${POD_CIDR} to any port ${port} proto tcp comment '${comment}' >/dev/null"
fi
}
ensure_ufw_rule 8899 k3s-pods-agave-rpc
ensure_ufw_rule 8900 k3s-pods-agave-ws
ensure_ufw_rule 10000 k3s-pods-agave-grpc
ssh_target "sudo systemctl status --no-pager agave-rpc-k3s.socket agave-ws-k3s.socket agave-grpc-k3s.socket | sed -n '1,80p'"
ssh_target "sudo ss -ltnp | egrep ':(8899|8900|10000)\\b' | sed -n '1,40p'"

View File

@@ -7,7 +7,7 @@ Minimal canary namespace for migration baseline `R001` on `sol`.
- Reserve a dedicated namespace for the first reconstructed trade deployment.
- Put hard upper bounds on namespace-level CPU, memory, object count, and PVC growth before application manifests land.
- Verify that workloads in the namespace can resolve and reach the shared `trade-infra` services for `Postgres` and `Redis`.
- Recreate the `R001` application surface in a controlled way: `Hasura`, `trade-api`, `trade-frontend`, and the first canary `trade-ingestor` path.
- Recreate the `R001` application surface in a controlled way: `Hasura`, `trade-api`, `trade-frontend`, the first canary `trade-ingestor` path, and the first DLOB hot-path components.
## Current Guardrails
@@ -34,16 +34,21 @@ Minimal canary namespace for migration baseline `R001` on `sol`.
- Current shared infrastructure endpoints expected by canary workloads:
- `postgres-host.trade-infra.svc.cluster.local:5432`
- `redis-host.trade-infra.svc.cluster.local:6379`
- `agave-rpc-host.trade-infra.svc.cluster.local:8899`
- `agave-ws-host.trade-infra.svc.cluster.local:8900`
- `agave-grpc-host.trade-infra.svc.cluster.local:10000`
## Application Surface
- `postgres` in this namespace is an `ExternalName` alias that points to `postgres-host.trade-infra.svc.cluster.local`.
- `Hasura` uses the live `R001` secrets copied from `trade-staging`, but connects to the host `Postgres` on `sol`.
- `trade-api` and `trade-frontend` use the current live images from Gitea registry and the same bootstrap wrapper/config pattern as the source environment.
- `dlob-publisher-hot` now targets the host validator on `sol` through `trade-infra` services and writes `dlob-hot:*` into the shared Redis host service.
- `dlob-hot-redis-to-postgres-raw-writer` and `dlob-hot-postgres-to-postgres-derived-writer` rebuild the first live DLOB derived path on `sol`.
- The canary workflow re-runs:
- `postgres-migrate`
- `hasura-bootstrap`
before it waits for `Hasura`, `trade-api`, `trade-frontend`, and `trade-ingestor` to become healthy.
before it waits for `Hasura`, `trade-api`, `trade-frontend`, `trade-ingestor`, and the DLOB hot-path deployments to become healthy.
- The current canary `trade-ingestor` is intentionally pinned to the schema already reconstructed on `sol` and reads from `dlob_stats_latest`.
- The exact live `R001` ingestor path that reads `dlob_*_derived_latest` remains a follow-up substep after the DLOB writer chain is reconstructed.
@@ -52,8 +57,11 @@ Minimal canary namespace for migration baseline `R001` on `sol`.
From the repository root:
```bash
./environments/sol/trade-infra/scripts/prepare-sol-agave-access.sh
kubectl apply -k environments/sol/trade-infra
./environments/sol/trade-r001-canary/scripts/prepare-sol-postgres.sh
./environments/sol/trade-r001-canary/scripts/create-gitea-registry-secret.sh
./environments/sol/trade-r001-canary/scripts/create-trade-dlob-rpc-secret.sh
./environments/sol/trade-r001-canary/scripts/sync-live-secrets.sh
```

View File

@@ -98,12 +98,17 @@ async function main() {
const source = 'default';
const baseTicks = { schema: 'public', name: 'drift_ticks' };
const dlobHotSnapshotLatestTable = { schema: 'public', name: 'dlob_hot_snapshot_latest' };
const dlobHotDerivedLatestTable = { schema: 'public', name: 'dlob_hot_derived_latest' };
const dlobAllDerivedLatestTable = { schema: 'public', name: 'dlob_all_derived_latest' };
const dlobL2LatestTable = { schema: 'public', name: 'dlob_l2_latest' };
const dlobStatsLatestTable = { schema: 'public', name: 'dlob_stats_latest' };
const dlobDepthBpsLatestTable = { schema: 'public', name: 'dlob_depth_bps_latest' };
const dlobSlippageLatestTable = { schema: 'public', name: 'dlob_slippage_latest' };
const dlobSlippageLatestV2Table = { schema: 'public', name: 'dlob_slippage_latest_v2' };
const candlesCacheTable = { schema: 'public', name: 'drift_candles_cache' };
const dlobHotDerivedTsTable = { schema: 'public', name: 'dlob_hot_derived_ts' };
const dlobAllDerivedTsTable = { schema: 'public', name: 'dlob_all_derived_ts' };
const dlobStatsTsTable = { schema: 'public', name: 'dlob_stats_ts' };
const dlobDepthBpsTsTable = { schema: 'public', name: 'dlob_depth_bps_ts' };
const dlobSlippageTsTable = { schema: 'public', name: 'dlob_slippage_ts' };
@@ -373,6 +378,171 @@ async function main() {
'updated_at',
], { publicFilter: dlobPublicFilter });
await ensurePublicSelectTable(dlobHotSnapshotLatestTable, [
'source',
'redis_key',
'snapshot_kind',
'market_type',
'market_index',
'market_name',
'is_indicative',
'ts_ms',
'slot',
'market_slot',
'payload_hash',
'mark_price_raw',
'oracle_price_raw',
'best_bid_price_raw',
'best_ask_price_raw',
'spread_pct_raw',
'spread_quote_raw',
'oracle_data',
'bids',
'asks',
'payload',
'updated_at',
]);
await ensurePublicSelectTable(dlobHotDerivedLatestTable, [
'source',
'market_type',
'market_index',
'market_name',
'is_indicative',
'ts_ms',
'slot',
'market_slot',
'mark_price',
'oracle_price',
'best_bid_price',
'best_ask_price',
'mid_price',
'spread_quote',
'spread_bps',
'depth_levels',
'bid_levels',
'ask_levels',
'top_bid_size',
'top_ask_size',
'top_bid_notional',
'top_ask_notional',
'depth_bid_base',
'depth_ask_base',
'depth_bid_quote',
'depth_ask_quote',
'imbalance',
'bids_norm',
'asks_norm',
'raw_payload_hash',
'updated_at',
]);
await ensurePublicSelectTable(dlobAllDerivedLatestTable, [
'source',
'market_type',
'market_index',
'market_name',
'is_indicative',
'ts_ms',
'slot',
'market_slot',
'mark_price',
'oracle_price',
'best_bid_price',
'best_ask_price',
'mid_price',
'spread_quote',
'spread_bps',
'depth_levels',
'bid_levels',
'ask_levels',
'top_bid_size',
'top_ask_size',
'top_bid_notional',
'top_ask_notional',
'depth_bid_base',
'depth_ask_base',
'depth_bid_quote',
'depth_ask_quote',
'imbalance',
'bids_norm',
'asks_norm',
'raw_payload_hash',
'updated_at',
]);
await ensurePublicSelectTable(dlobHotDerivedTsTable, [
'event_ts',
'id',
'source',
'market_type',
'market_index',
'market_name',
'is_indicative',
'ts_ms',
'slot',
'market_slot',
'mark_price',
'oracle_price',
'best_bid_price',
'best_ask_price',
'mid_price',
'spread_quote',
'spread_bps',
'depth_levels',
'bid_levels',
'ask_levels',
'top_bid_size',
'top_ask_size',
'top_bid_notional',
'top_ask_notional',
'depth_bid_base',
'depth_ask_base',
'depth_bid_quote',
'depth_ask_quote',
'imbalance',
'bids_norm',
'asks_norm',
'raw_payload_hash',
'inserted_at',
]);
await ensurePublicSelectTable(dlobAllDerivedTsTable, [
'event_ts',
'id',
'source',
'market_type',
'market_index',
'market_name',
'is_indicative',
'ts_ms',
'slot',
'market_slot',
'mark_price',
'oracle_price',
'best_bid_price',
'best_ask_price',
'mid_price',
'spread_quote',
'spread_bps',
'depth_levels',
'bid_levels',
'ask_levels',
'top_bid_size',
'top_ask_size',
'top_bid_notional',
'top_ask_notional',
'depth_bid_base',
'depth_ask_base',
'depth_bid_quote',
'depth_ask_quote',
'imbalance',
'bids_norm',
'asks_norm',
'raw_payload_hash',
'inserted_at',
]);
await ensurePublicSelectTable(dlobStatsTsTable, [
'ts',
'id',

View File

@@ -858,3 +858,271 @@ BEGIN
PERFORM add_retention_policy('dlob_slippage_ts_v2', INTERVAL '7 days');
EXCEPTION WHEN OTHERS THEN
END $$;
-- Canonical raw DLOB hot snapshots written from Redis to PostgreSQL.
CREATE TABLE IF NOT EXISTS public.dlob_hot_snapshot_latest (
source TEXT NOT NULL,
redis_key TEXT NOT NULL,
snapshot_kind TEXT NOT NULL CHECK (snapshot_kind IN ('orderbook_l2', 'orderbook_l3', 'best_makers')),
market_type TEXT NOT NULL,
market_index INTEGER NOT NULL,
market_name TEXT NOT NULL,
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
ts_ms BIGINT NOT NULL,
slot BIGINT,
market_slot BIGINT,
payload_hash TEXT NOT NULL,
mark_price_raw TEXT,
oracle_price_raw TEXT,
best_bid_price_raw TEXT,
best_ask_price_raw TEXT,
spread_pct_raw TEXT,
spread_quote_raw TEXT,
oracle_data JSONB,
bids JSONB,
asks JSONB,
payload JSONB NOT NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (source, market_type, market_index, snapshot_kind, is_indicative)
);
CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_latest_updated_at_idx
ON public.dlob_hot_snapshot_latest (updated_at DESC);
CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_latest_source_market_idx
ON public.dlob_hot_snapshot_latest (source, market_type, market_index, updated_at DESC);
CREATE TABLE IF NOT EXISTS public.dlob_hot_snapshot_ts (
event_ts TIMESTAMPTZ NOT NULL,
id BIGSERIAL NOT NULL,
source TEXT NOT NULL,
redis_key TEXT NOT NULL,
snapshot_kind TEXT NOT NULL CHECK (snapshot_kind IN ('orderbook_l2', 'orderbook_l3', 'best_makers')),
market_type TEXT NOT NULL,
market_index INTEGER NOT NULL,
market_name TEXT NOT NULL,
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
ts_ms BIGINT NOT NULL,
slot BIGINT,
market_slot BIGINT,
payload_hash TEXT NOT NULL,
mark_price_raw TEXT,
oracle_price_raw TEXT,
best_bid_price_raw TEXT,
best_ask_price_raw TEXT,
spread_pct_raw TEXT,
spread_quote_raw TEXT,
oracle_data JSONB,
bids JSONB,
asks JSONB,
payload JSONB NOT NULL,
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (event_ts, id)
);
SELECT create_hypertable('dlob_hot_snapshot_ts', 'event_ts', if_not_exists => TRUE, migrate_data => TRUE);
CREATE UNIQUE INDEX IF NOT EXISTS dlob_hot_snapshot_ts_dedupe_idx
ON public.dlob_hot_snapshot_ts (event_ts, source, redis_key, payload_hash);
CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_ts_market_ts_desc_idx
ON public.dlob_hot_snapshot_ts (market_type, market_index, event_ts DESC);
CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_ts_source_market_ts_desc_idx
ON public.dlob_hot_snapshot_ts (source, market_type, market_index, event_ts DESC);
CREATE TABLE IF NOT EXISTS public.dlob_hot_derived_latest (
source TEXT NOT NULL,
market_type TEXT NOT NULL,
market_index INTEGER NOT NULL,
market_name TEXT NOT NULL,
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
ts_ms BIGINT NOT NULL,
slot BIGINT,
market_slot BIGINT,
mark_price NUMERIC,
oracle_price NUMERIC,
best_bid_price NUMERIC,
best_ask_price NUMERIC,
mid_price NUMERIC,
spread_quote NUMERIC,
spread_bps NUMERIC,
depth_levels INTEGER NOT NULL,
bid_levels INTEGER NOT NULL,
ask_levels INTEGER NOT NULL,
top_bid_size NUMERIC,
top_ask_size NUMERIC,
top_bid_notional NUMERIC,
top_ask_notional NUMERIC,
depth_bid_base NUMERIC NOT NULL DEFAULT 0,
depth_ask_base NUMERIC NOT NULL DEFAULT 0,
depth_bid_quote NUMERIC NOT NULL DEFAULT 0,
depth_ask_quote NUMERIC NOT NULL DEFAULT 0,
imbalance NUMERIC,
bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
raw_payload_hash TEXT NOT NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (source, market_type, market_index, is_indicative)
);
CREATE INDEX IF NOT EXISTS dlob_hot_derived_latest_updated_at_idx
ON public.dlob_hot_derived_latest (updated_at DESC);
CREATE INDEX IF NOT EXISTS dlob_hot_derived_latest_source_market_idx
ON public.dlob_hot_derived_latest (source, market_type, market_index, updated_at DESC);
CREATE TABLE IF NOT EXISTS public.dlob_hot_derived_ts (
event_ts TIMESTAMPTZ NOT NULL,
id BIGSERIAL NOT NULL,
source TEXT NOT NULL,
market_type TEXT NOT NULL,
market_index INTEGER NOT NULL,
market_name TEXT NOT NULL,
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
ts_ms BIGINT NOT NULL,
slot BIGINT,
market_slot BIGINT,
mark_price NUMERIC,
oracle_price NUMERIC,
best_bid_price NUMERIC,
best_ask_price NUMERIC,
mid_price NUMERIC,
spread_quote NUMERIC,
spread_bps NUMERIC,
depth_levels INTEGER NOT NULL,
bid_levels INTEGER NOT NULL,
ask_levels INTEGER NOT NULL,
top_bid_size NUMERIC,
top_ask_size NUMERIC,
top_bid_notional NUMERIC,
top_ask_notional NUMERIC,
depth_bid_base NUMERIC NOT NULL DEFAULT 0,
depth_ask_base NUMERIC NOT NULL DEFAULT 0,
depth_bid_quote NUMERIC NOT NULL DEFAULT 0,
depth_ask_quote NUMERIC NOT NULL DEFAULT 0,
imbalance NUMERIC,
bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
raw_payload_hash TEXT NOT NULL,
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (event_ts, id)
);
SELECT create_hypertable('dlob_hot_derived_ts', 'event_ts', if_not_exists => TRUE, migrate_data => TRUE);
CREATE UNIQUE INDEX IF NOT EXISTS dlob_hot_derived_ts_dedupe_idx
ON public.dlob_hot_derived_ts (event_ts, source, market_type, market_index, is_indicative, raw_payload_hash);
CREATE INDEX IF NOT EXISTS dlob_hot_derived_ts_market_ts_desc_idx
ON public.dlob_hot_derived_ts (market_type, market_index, event_ts DESC);
CREATE INDEX IF NOT EXISTS dlob_hot_derived_ts_source_market_ts_desc_idx
ON public.dlob_hot_derived_ts (source, market_type, market_index, event_ts DESC);
DO $$
BEGIN
PERFORM add_retention_policy('dlob_hot_snapshot_ts', INTERVAL '30 days');
EXCEPTION WHEN duplicate_object THEN NULL;
END $$;
DO $$
BEGIN
PERFORM add_retention_policy('dlob_hot_derived_ts', INTERVAL '180 days');
EXCEPTION WHEN duplicate_object THEN NULL;
END $$;
CREATE TABLE IF NOT EXISTS public.dlob_all_derived_latest (
source TEXT NOT NULL,
market_type TEXT NOT NULL,
market_index INTEGER NOT NULL,
market_name TEXT NOT NULL,
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
ts_ms BIGINT NOT NULL,
slot BIGINT,
market_slot BIGINT,
mark_price NUMERIC,
oracle_price NUMERIC,
best_bid_price NUMERIC,
best_ask_price NUMERIC,
mid_price NUMERIC,
spread_quote NUMERIC,
spread_bps NUMERIC,
depth_levels INTEGER NOT NULL,
bid_levels INTEGER NOT NULL,
ask_levels INTEGER NOT NULL,
top_bid_size NUMERIC,
top_ask_size NUMERIC,
top_bid_notional NUMERIC,
top_ask_notional NUMERIC,
depth_bid_base NUMERIC NOT NULL DEFAULT 0,
depth_ask_base NUMERIC NOT NULL DEFAULT 0,
depth_bid_quote NUMERIC NOT NULL DEFAULT 0,
depth_ask_quote NUMERIC NOT NULL DEFAULT 0,
imbalance NUMERIC,
bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
raw_payload_hash TEXT NOT NULL,
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (source, market_type, market_index, is_indicative)
);
CREATE INDEX IF NOT EXISTS dlob_all_derived_latest_updated_at_idx
ON public.dlob_all_derived_latest (updated_at DESC);
CREATE INDEX IF NOT EXISTS dlob_all_derived_latest_source_market_idx
ON public.dlob_all_derived_latest (source, market_type, market_index, updated_at DESC);
CREATE TABLE IF NOT EXISTS public.dlob_all_derived_ts (
event_ts TIMESTAMPTZ NOT NULL,
id BIGSERIAL NOT NULL,
source TEXT NOT NULL,
market_type TEXT NOT NULL,
market_index INTEGER NOT NULL,
market_name TEXT NOT NULL,
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
ts_ms BIGINT NOT NULL,
slot BIGINT,
market_slot BIGINT,
mark_price NUMERIC,
oracle_price NUMERIC,
best_bid_price NUMERIC,
best_ask_price NUMERIC,
mid_price NUMERIC,
spread_quote NUMERIC,
spread_bps NUMERIC,
depth_levels INTEGER NOT NULL,
bid_levels INTEGER NOT NULL,
ask_levels INTEGER NOT NULL,
top_bid_size NUMERIC,
top_ask_size NUMERIC,
top_bid_notional NUMERIC,
top_ask_notional NUMERIC,
depth_bid_base NUMERIC NOT NULL DEFAULT 0,
depth_ask_base NUMERIC NOT NULL DEFAULT 0,
depth_bid_quote NUMERIC NOT NULL DEFAULT 0,
depth_ask_quote NUMERIC NOT NULL DEFAULT 0,
imbalance NUMERIC,
bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
raw_payload_hash TEXT NOT NULL,
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
PRIMARY KEY (event_ts, id)
);
SELECT create_hypertable('dlob_all_derived_ts', 'event_ts', if_not_exists => TRUE, migrate_data => TRUE);
CREATE UNIQUE INDEX IF NOT EXISTS dlob_all_derived_ts_dedupe_idx
ON public.dlob_all_derived_ts (event_ts, source, market_type, market_index, is_indicative, raw_payload_hash);
CREATE INDEX IF NOT EXISTS dlob_all_derived_ts_market_ts_desc_idx
ON public.dlob_all_derived_ts (market_type, market_index, event_ts DESC);
CREATE INDEX IF NOT EXISTS dlob_all_derived_ts_source_market_ts_desc_idx
ON public.dlob_all_derived_ts (source, market_type, market_index, event_ts DESC);
DO $$
BEGIN
PERFORM add_retention_policy('dlob_all_derived_ts', INTERVAL '30 days');
EXCEPTION WHEN duplicate_object THEN NULL;
END $$;

View File

@@ -0,0 +1,63 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-hot-postgres-to-postgres-derived-writer
namespace: trade-r001-canary
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-hot-postgres-to-postgres-derived-writer
template:
metadata:
labels:
app.kubernetes.io/name: dlob-hot-postgres-to-postgres-derived-writer
spec:
imagePullSecrets:
- name: gitea-registry
containers:
- name: writer
image: gitea.mpabi.pl/trade/trade-dlob-server:hot-pg-events-20260320-205517
imagePullPolicy: IfNotPresent
command:
- node
- /lib/scripts/dlobHotPostgresToPostgresDerivedWriter.js
env:
- name: DLOB_SOURCE
value: mevnode_bot_hot_derived
- name: RAW_SOURCE
value: mevnode_bot_hot_raw
- name: DLOB_POLL_MS
value: "5000"
- name: NORMALIZED_DEPTH
value: "10"
- name: PRICE_PRECISION
value: "1000000"
- name: BASE_PRECISION
value: "1000000000"
- name: PGHOST
value: postgres
- name: PGPORT
value: "5432"
- name: PGUSER
valueFrom:
secretKeyRef:
name: trade-postgres
key: POSTGRES_USER
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: trade-postgres
key: POSTGRES_PASSWORD
- name: PGDATABASE
valueFrom:
secretKeyRef:
name: trade-postgres
key: POSTGRES_DB
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 250m
memory: 256Mi

View File

@@ -0,0 +1,61 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-hot-redis-to-postgres-raw-writer
namespace: trade-r001-canary
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-hot-redis-to-postgres-raw-writer
template:
metadata:
labels:
app.kubernetes.io/name: dlob-hot-redis-to-postgres-raw-writer
spec:
imagePullSecrets:
- name: gitea-registry
containers:
- name: writer
image: gitea.mpabi.pl/trade/trade-dlob-server:hot-pg-events-20260320-205517
imagePullPolicy: IfNotPresent
command:
- node
- /lib/scripts/dlobHotRedisToPostgresRawWriter.js
env:
- name: DLOB_SOURCE
value: mevnode_bot_hot_raw
- name: REDIS_HOST
value: dlob-redis
- name: REDIS_PORT
value: "6379"
- name: REDIS_KEY_PREFIX
value: "dlob-hot:"
- name: DLOB_POLL_MS
value: "5000"
- name: PGHOST
value: postgres
- name: PGPORT
value: "5432"
- name: PGUSER
valueFrom:
secretKeyRef:
name: trade-postgres
key: POSTGRES_USER
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: trade-postgres
key: POSTGRES_PASSWORD
- name: PGDATABASE
valueFrom:
secretKeyRef:
name: trade-postgres
key: POSTGRES_DB
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 250m
memory: 256Mi

View File

@@ -0,0 +1,130 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: dlob-publisher-hot
namespace: trade-r001-canary
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-publisher-hot
template:
metadata:
labels:
app.kubernetes.io/name: dlob-publisher-hot
spec:
imagePullSecrets:
- name: gitea-registry
containers:
- name: publisher
image: gitea.mpabi.pl/trade/trade-dlob-server:grpc-teardown-fix-20260402-113736
imagePullPolicy: IfNotPresent
command:
- node
- /lib/publishers/dlobPublisher.js
ports:
- name: http
containerPort: 8080
env:
- name: FETCH_CONNECT_TIMEOUT_MS
value: "15000"
- name: FETCH_HEADERS_TIMEOUT_MS
value: "300000"
- name: FETCH_BODY_TIMEOUT_MS
value: "300000"
- name: ENABLE_PERSISTENT_STORE
value: "true"
- name: DLOB_SOURCE
value: mevnode_bot_hot
- name: PRICE_PRECISION
value: "1000000"
- name: BASE_PRECISION
value: "1000000000"
- name: PERSISTENT_STATS_DEPTH
value: "10"
- name: PGHOST
value: postgres
- name: PGPORT
value: "5432"
- name: PGUSER
valueFrom:
secretKeyRef:
name: trade-postgres
key: POSTGRES_USER
- name: PGPASSWORD
valueFrom:
secretKeyRef:
name: trade-postgres
key: POSTGRES_PASSWORD
- name: PGDATABASE
valueFrom:
secretKeyRef:
name: trade-postgres
key: POSTGRES_DB
- name: PERP_MARKETS_TO_LOAD
value: "0,20,72"
- name: USE_GRPC
value: "true"
- name: USE_WEBSOCKET
value: "true"
- name: DISABLE_GPA_REFRESH
value: "true"
- name: GRPC_CLIENT
value: yellowstone
- name: GRPC_ENDPOINT
valueFrom:
secretKeyRef:
name: trade-dlob-rpc
key: GRPC_ENDPOINT
- name: TOKEN
valueFrom:
secretKeyRef:
name: trade-dlob-rpc
key: TOKEN
- name: RUNNING_LOCAL
value: "true"
- name: LOCAL_CACHE
value: "true"
- name: ENV
value: mainnet-beta
- name: USE_ORDER_SUBSCRIBER
value: "true"
- name: ELASTICACHE_HOST
value: dlob-redis
- name: ELASTICACHE_PORT
value: "6379"
- name: REDIS_CLIENT
value: DLOB_HOT
- name: ENDPOINT
valueFrom:
secretKeyRef:
name: trade-dlob-rpc
key: ENDPOINT
- name: WS_ENDPOINT
valueFrom:
secretKeyRef:
name: trade-dlob-rpc
key: WS_ENDPOINT
resources:
requests:
cpu: 250m
memory: 512Mi
limits:
cpu: "1"
memory: 1Gi
readinessProbe:
httpGet:
path: /startup
port: http
initialDelaySeconds: 120
periodSeconds: 10
timeoutSeconds: 3
failureThreshold: 30
livenessProbe:
httpGet:
path: /health
port: http
initialDelaySeconds: 240
periodSeconds: 20
timeoutSeconds: 3
failureThreshold: 10

View File

@@ -0,0 +1,12 @@
apiVersion: v1
kind: Service
metadata:
name: dlob-redis
namespace: trade-r001-canary
spec:
type: ExternalName
externalName: redis-host.trade-infra.svc.cluster.local
ports:
- name: redis
port: 6379
targetPort: 6379

View File

@@ -15,6 +15,13 @@ spec:
envFrom:
- secretRef:
name: trade-postgres
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 250m
memory: 256Mi
command:
- sh
- -ec
@@ -38,6 +45,13 @@ spec:
value: drift_ticks
- name: CANDLES_FUNCTION
value: get_drift_candles
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 250m
memory: 256Mi
command:
- node
- /app/hasura-bootstrap.mjs

View File

@@ -11,6 +11,7 @@ resources:
- resourcequota.yaml
- limitrange.yaml
- postgres-alias-service.yaml
- dlob-redis-alias-service.yaml
- hasura-service.yaml
- hasura-deployment.yaml
- postgres-migrate-job.yaml
@@ -20,6 +21,9 @@ resources:
- trade-frontend-service.yaml
- trade-frontend-deployment.yaml
- trade-ingestor-deployment.yaml
- dlob-publisher-hot-deployment.yaml
- dlob-hot-redis-to-postgres-raw-writer-deployment.yaml
- dlob-hot-postgres-to-postgres-derived-writer-deployment.yaml
configMapGenerator:
- name: postgres-initdb

View File

@@ -15,6 +15,13 @@ spec:
envFrom:
- secretRef:
name: trade-postgres
resources:
requests:
cpu: 100m
memory: 128Mi
limits:
cpu: 250m
memory: 256Mi
command:
- sh
- -ec

View File

@@ -0,0 +1,25 @@
#!/usr/bin/env bash
set -euo pipefail
TARGET_HOST="${TARGET_HOST:-mevnode}"
TARGET_NAMESPACE="${TARGET_NAMESPACE:-trade-r001-canary}"
RPC_URL="${RPC_URL:-http://agave-rpc-host.trade-infra.svc.cluster.local:8899}"
WS_URL="${WS_URL:-ws://agave-ws-host.trade-infra.svc.cluster.local:8900}"
GRPC_URL="${GRPC_URL:-http://agave-grpc-host.trade-infra.svc.cluster.local:10000}"
ssh_target() {
ssh -o StrictHostKeyChecking=no "$TARGET_HOST" "$@"
}
TOKEN="$(ssh_target 'sudo cat /etc/agave/geyser.x_token')"
ssh_target "sudo k3s kubectl get ns ${TARGET_NAMESPACE} >/dev/null 2>&1 || sudo k3s kubectl create ns ${TARGET_NAMESPACE} >/dev/null"
ssh_target "sudo k3s kubectl -n ${TARGET_NAMESPACE} create secret generic trade-dlob-rpc \
--from-literal=ENDPOINT='${RPC_URL}' \
--from-literal=WS_ENDPOINT='${WS_URL}' \
--from-literal=GRPC_ENDPOINT='${GRPC_URL}' \
--from-literal=TOKEN='${TOKEN}' \
--dry-run=client -o yaml | sudo k3s kubectl apply -f - >/dev/null"
echo "Created trade-dlob-rpc in ${TARGET_NAMESPACE}"