feat(sol): add agave-backed dlob hot path for canary
Some checks failed
deploy-trade-r001-canary / apply (push) Failing after 5m41s
Some checks failed
deploy-trade-r001-canary / apply (push) Failing after 5m41s
This commit is contained in:
@@ -34,7 +34,7 @@ jobs:
|
||||
env:
|
||||
KUBECONFIG: /tmp/kubeconfig
|
||||
run: |
|
||||
kubectl -n trade-r001-canary get secret trade-postgres trade-hasura trade-api trade-frontend-tokens trade-basic-auth trade-ingestor-tokens gitea-registry
|
||||
kubectl -n trade-r001-canary get secret trade-postgres trade-hasura trade-api trade-frontend-tokens trade-basic-auth trade-ingestor-tokens trade-dlob-rpc gitea-registry
|
||||
|
||||
- name: Recreate bootstrap jobs
|
||||
env:
|
||||
@@ -42,6 +42,13 @@ jobs:
|
||||
run: |
|
||||
kubectl -n trade-r001-canary delete job postgres-migrate hasura-bootstrap --ignore-not-found=true
|
||||
|
||||
- name: Apply shared host access infrastructure
|
||||
env:
|
||||
KUBECONFIG: /tmp/kubeconfig
|
||||
run: |
|
||||
kubectl apply -k environments/sol/trade-infra
|
||||
kubectl -n trade-infra get svc,endpointslice
|
||||
|
||||
- name: Apply canary environment
|
||||
env:
|
||||
KUBECONFIG: /tmp/kubeconfig
|
||||
@@ -54,7 +61,14 @@ jobs:
|
||||
env:
|
||||
KUBECONFIG: /tmp/kubeconfig
|
||||
run: |
|
||||
kubectl -n trade-r001-canary rollout restart deploy/hasura deploy/trade-api deploy/trade-frontend deploy/trade-ingestor
|
||||
kubectl -n trade-r001-canary rollout restart \
|
||||
deploy/hasura \
|
||||
deploy/trade-api \
|
||||
deploy/trade-frontend \
|
||||
deploy/trade-ingestor \
|
||||
deploy/dlob-publisher-hot \
|
||||
deploy/dlob-hot-redis-to-postgres-raw-writer \
|
||||
deploy/dlob-hot-postgres-to-postgres-derived-writer
|
||||
|
||||
- name: Wait for database and metadata bootstrap
|
||||
env:
|
||||
@@ -71,6 +85,9 @@ jobs:
|
||||
kubectl -n trade-r001-canary rollout status deploy/trade-api --timeout=300s
|
||||
kubectl -n trade-r001-canary rollout status deploy/trade-frontend --timeout=300s
|
||||
kubectl -n trade-r001-canary rollout status deploy/trade-ingestor --timeout=300s
|
||||
kubectl -n trade-r001-canary rollout status deploy/dlob-publisher-hot --timeout=420s
|
||||
kubectl -n trade-r001-canary rollout status deploy/dlob-hot-redis-to-postgres-raw-writer --timeout=300s
|
||||
kubectl -n trade-r001-canary rollout status deploy/dlob-hot-postgres-to-postgres-derived-writer --timeout=300s
|
||||
kubectl -n trade-r001-canary get deploy,pods -o wide
|
||||
|
||||
- name: Verify trade-ingestor runtime
|
||||
@@ -94,6 +111,9 @@ jobs:
|
||||
const targets = [
|
||||
['postgres-host.trade-infra.svc.cluster.local', 5432],
|
||||
['redis-host.trade-infra.svc.cluster.local', 6379],
|
||||
['agave-rpc-host.trade-infra.svc.cluster.local', 8899],
|
||||
['agave-ws-host.trade-infra.svc.cluster.local', 8900],
|
||||
['agave-grpc-host.trade-infra.svc.cluster.local', 10000],
|
||||
];
|
||||
|
||||
function checkSocket(host, port) {
|
||||
@@ -120,6 +140,63 @@ jobs:
|
||||
process.exit(1);
|
||||
});
|
||||
JS
|
||||
|
||||
- name: Verify DLOB hot-path runtime
|
||||
env:
|
||||
KUBECONFIG: /tmp/kubeconfig
|
||||
run: |
|
||||
kubectl -n trade-r001-canary logs deploy/dlob-publisher-hot --tail=20
|
||||
kubectl -n trade-r001-canary logs deploy/dlob-hot-redis-to-postgres-raw-writer --tail=20
|
||||
kubectl -n trade-r001-canary logs deploy/dlob-hot-postgres-to-postgres-derived-writer --tail=20
|
||||
pod_name="$(kubectl -n trade-r001-canary get pod -l app.kubernetes.io/name=trade-ingestor -o jsonpath='{.items[0].metadata.name}')"
|
||||
kubectl -n trade-r001-canary exec -i "$pod_name" -- node - <<'JS'
|
||||
const endpoint = 'http://hasura:8080/v1/graphql';
|
||||
const adminSecret = process.env.HASURA_ADMIN_SECRET;
|
||||
const query = `
|
||||
query {
|
||||
dlob_hot_derived_latest(limit: 1, order_by: { updated_at: desc }) {
|
||||
market_name
|
||||
updated_at
|
||||
source
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
async function check() {
|
||||
const response = await fetch(endpoint, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'content-type': 'application/json',
|
||||
'x-hasura-admin-secret': adminSecret,
|
||||
},
|
||||
body: JSON.stringify({ query }),
|
||||
signal: AbortSignal.timeout(10000),
|
||||
});
|
||||
const payload = await response.json();
|
||||
const rows = payload?.data?.dlob_hot_derived_latest || [];
|
||||
if (!rows.length) {
|
||||
throw new Error('No rows in dlob_hot_derived_latest yet');
|
||||
}
|
||||
console.log(JSON.stringify(rows[0], null, 2));
|
||||
}
|
||||
|
||||
(async () => {
|
||||
for (let attempt = 0; attempt < 24; attempt += 1) {
|
||||
try {
|
||||
await check();
|
||||
return;
|
||||
} catch (error) {
|
||||
if (attempt === 23) {
|
||||
throw error;
|
||||
}
|
||||
await new Promise((resolve) => setTimeout(resolve, 5000));
|
||||
}
|
||||
}
|
||||
})().catch((error) => {
|
||||
console.error(String(error && error.message ? error.message : error));
|
||||
process.exit(1);
|
||||
});
|
||||
JS
|
||||
kubectl -n trade-r001-canary exec -i "$pod_name" -- node - <<'JS'
|
||||
const targets = [
|
||||
'http://hasura:8080/healthz',
|
||||
|
||||
25
environments/sol/trade-infra/README.md
Normal file
25
environments/sol/trade-infra/README.md
Normal file
@@ -0,0 +1,25 @@
|
||||
# trade-infra
|
||||
|
||||
Shared host-backed services for the `sol` cluster.
|
||||
|
||||
## Purpose
|
||||
|
||||
- Expose host services into `k3s` through stable service names in namespace `trade-infra`.
|
||||
- Keep host access paths reproducible in Git instead of relying on manual `kubectl` history.
|
||||
- Provide cluster DNS names for:
|
||||
- `Postgres`
|
||||
- `Redis`
|
||||
- `agave` RPC
|
||||
- `agave` websocket
|
||||
- `agave` Yellowstone gRPC
|
||||
|
||||
## Operator Flow
|
||||
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
./environments/sol/trade-infra/scripts/prepare-sol-agave-access.sh
|
||||
kubectl apply -k environments/sol/trade-infra
|
||||
```
|
||||
|
||||
`prepare-sol-agave-access.sh` installs host-level socket proxies on `sol` so pods can reach the private validator endpoints through the host IP `149.50.96.162`.
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: discovery.k8s.io/v1
|
||||
kind: EndpointSlice
|
||||
metadata:
|
||||
name: agave-grpc-host-sol
|
||||
namespace: trade-infra
|
||||
labels:
|
||||
kubernetes.io/service-name: agave-grpc-host
|
||||
addressType: IPv4
|
||||
ports:
|
||||
- name: grpc
|
||||
protocol: TCP
|
||||
port: 10000
|
||||
endpoints:
|
||||
- addresses:
|
||||
- 149.50.96.162
|
||||
10
environments/sol/trade-infra/agave-grpc-host-service.yaml
Normal file
10
environments/sol/trade-infra/agave-grpc-host-service.yaml
Normal file
@@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: agave-grpc-host
|
||||
namespace: trade-infra
|
||||
spec:
|
||||
ports:
|
||||
- name: grpc
|
||||
port: 10000
|
||||
targetPort: 10000
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: discovery.k8s.io/v1
|
||||
kind: EndpointSlice
|
||||
metadata:
|
||||
name: agave-rpc-host-sol
|
||||
namespace: trade-infra
|
||||
labels:
|
||||
kubernetes.io/service-name: agave-rpc-host
|
||||
addressType: IPv4
|
||||
ports:
|
||||
- name: rpc
|
||||
protocol: TCP
|
||||
port: 8899
|
||||
endpoints:
|
||||
- addresses:
|
||||
- 149.50.96.162
|
||||
10
environments/sol/trade-infra/agave-rpc-host-service.yaml
Normal file
10
environments/sol/trade-infra/agave-rpc-host-service.yaml
Normal file
@@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: agave-rpc-host
|
||||
namespace: trade-infra
|
||||
spec:
|
||||
ports:
|
||||
- name: rpc
|
||||
port: 8899
|
||||
targetPort: 8899
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: discovery.k8s.io/v1
|
||||
kind: EndpointSlice
|
||||
metadata:
|
||||
name: agave-ws-host-sol
|
||||
namespace: trade-infra
|
||||
labels:
|
||||
kubernetes.io/service-name: agave-ws-host
|
||||
addressType: IPv4
|
||||
ports:
|
||||
- name: ws
|
||||
protocol: TCP
|
||||
port: 8900
|
||||
endpoints:
|
||||
- addresses:
|
||||
- 149.50.96.162
|
||||
10
environments/sol/trade-infra/agave-ws-host-service.yaml
Normal file
10
environments/sol/trade-infra/agave-ws-host-service.yaml
Normal file
@@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: agave-ws-host
|
||||
namespace: trade-infra
|
||||
spec:
|
||||
ports:
|
||||
- name: ws
|
||||
port: 8900
|
||||
targetPort: 8900
|
||||
17
environments/sol/trade-infra/kustomization.yaml
Normal file
17
environments/sol/trade-infra/kustomization.yaml
Normal file
@@ -0,0 +1,17 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: trade-infra
|
||||
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- postgres-host-service.yaml
|
||||
- postgres-host-endpointslice.yaml
|
||||
- redis-host-service.yaml
|
||||
- redis-host-endpointslice.yaml
|
||||
- agave-rpc-host-service.yaml
|
||||
- agave-rpc-host-endpointslice.yaml
|
||||
- agave-ws-host-service.yaml
|
||||
- agave-ws-host-endpointslice.yaml
|
||||
- agave-grpc-host-service.yaml
|
||||
- agave-grpc-host-endpointslice.yaml
|
||||
4
environments/sol/trade-infra/namespace.yaml
Normal file
4
environments/sol/trade-infra/namespace.yaml
Normal file
@@ -0,0 +1,4 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: trade-infra
|
||||
@@ -0,0 +1,15 @@
|
||||
apiVersion: discovery.k8s.io/v1
|
||||
kind: EndpointSlice
|
||||
metadata:
|
||||
name: postgres-host-sol
|
||||
namespace: trade-infra
|
||||
labels:
|
||||
kubernetes.io/service-name: postgres-host
|
||||
addressType: IPv4
|
||||
ports:
|
||||
- name: postgres
|
||||
protocol: TCP
|
||||
port: 5432
|
||||
endpoints:
|
||||
- addresses:
|
||||
- 149.50.96.162
|
||||
10
environments/sol/trade-infra/postgres-host-service.yaml
Normal file
10
environments/sol/trade-infra/postgres-host-service.yaml
Normal file
@@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: postgres-host
|
||||
namespace: trade-infra
|
||||
spec:
|
||||
ports:
|
||||
- name: postgres
|
||||
port: 5432
|
||||
targetPort: 5432
|
||||
15
environments/sol/trade-infra/redis-host-endpointslice.yaml
Normal file
15
environments/sol/trade-infra/redis-host-endpointslice.yaml
Normal file
@@ -0,0 +1,15 @@
|
||||
apiVersion: discovery.k8s.io/v1
|
||||
kind: EndpointSlice
|
||||
metadata:
|
||||
name: redis-host-sol
|
||||
namespace: trade-infra
|
||||
labels:
|
||||
kubernetes.io/service-name: redis-host
|
||||
addressType: IPv4
|
||||
ports:
|
||||
- name: redis
|
||||
protocol: TCP
|
||||
port: 6379
|
||||
endpoints:
|
||||
- addresses:
|
||||
- 149.50.96.162
|
||||
10
environments/sol/trade-infra/redis-host-service.yaml
Normal file
10
environments/sol/trade-infra/redis-host-service.yaml
Normal file
@@ -0,0 +1,10 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: redis-host
|
||||
namespace: trade-infra
|
||||
spec:
|
||||
ports:
|
||||
- name: redis
|
||||
port: 6379
|
||||
targetPort: 6379
|
||||
96
environments/sol/trade-infra/scripts/prepare-sol-agave-access.sh
Executable file
96
environments/sol/trade-infra/scripts/prepare-sol-agave-access.sh
Executable file
@@ -0,0 +1,96 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
TARGET_HOST="${TARGET_HOST:-mevnode}"
|
||||
HOST_IP="${HOST_IP:-149.50.96.162}"
|
||||
POD_CIDR="${POD_CIDR:-10.42.0.0/24}"
|
||||
|
||||
ssh_target() {
|
||||
ssh -o StrictHostKeyChecking=no "$TARGET_HOST" "$@"
|
||||
}
|
||||
|
||||
install_unit() {
|
||||
local unit_path="$1"
|
||||
ssh_target "sudo tee ${unit_path} >/dev/null"
|
||||
}
|
||||
|
||||
cat <<EOF | install_unit /etc/systemd/system/agave-rpc-k3s.socket
|
||||
[Unit]
|
||||
Description=Expose Agave RPC on host IP for k3s pods
|
||||
|
||||
[Socket]
|
||||
ListenStream=${HOST_IP}:8899
|
||||
NoDelay=true
|
||||
|
||||
[Install]
|
||||
WantedBy=sockets.target
|
||||
EOF
|
||||
|
||||
cat <<'EOF' | install_unit /etc/systemd/system/agave-rpc-k3s.service
|
||||
[Unit]
|
||||
Description=Proxy Agave RPC from host IP to localhost
|
||||
|
||||
[Service]
|
||||
ExecStart=/lib/systemd/systemd-socket-proxyd 127.0.0.1:8899
|
||||
PrivateNetwork=no
|
||||
EOF
|
||||
|
||||
cat <<EOF | install_unit /etc/systemd/system/agave-ws-k3s.socket
|
||||
[Unit]
|
||||
Description=Expose Agave websocket on host IP for k3s pods
|
||||
|
||||
[Socket]
|
||||
ListenStream=${HOST_IP}:8900
|
||||
NoDelay=true
|
||||
|
||||
[Install]
|
||||
WantedBy=sockets.target
|
||||
EOF
|
||||
|
||||
cat <<'EOF' | install_unit /etc/systemd/system/agave-ws-k3s.service
|
||||
[Unit]
|
||||
Description=Proxy Agave websocket from host IP to localhost
|
||||
|
||||
[Service]
|
||||
ExecStart=/lib/systemd/systemd-socket-proxyd 127.0.0.1:8900
|
||||
PrivateNetwork=no
|
||||
EOF
|
||||
|
||||
cat <<EOF | install_unit /etc/systemd/system/agave-grpc-k3s.socket
|
||||
[Unit]
|
||||
Description=Expose Agave Yellowstone gRPC on host IP for k3s pods
|
||||
|
||||
[Socket]
|
||||
ListenStream=${HOST_IP}:10000
|
||||
NoDelay=true
|
||||
|
||||
[Install]
|
||||
WantedBy=sockets.target
|
||||
EOF
|
||||
|
||||
cat <<'EOF' | install_unit /etc/systemd/system/agave-grpc-k3s.service
|
||||
[Unit]
|
||||
Description=Proxy Agave Yellowstone gRPC from host IP to WireGuard IP
|
||||
|
||||
[Service]
|
||||
ExecStart=/lib/systemd/systemd-socket-proxyd 10.91.0.1:10000
|
||||
PrivateNetwork=no
|
||||
EOF
|
||||
|
||||
ssh_target "sudo systemctl daemon-reload"
|
||||
ssh_target "sudo systemctl enable --now agave-rpc-k3s.socket agave-ws-k3s.socket agave-grpc-k3s.socket"
|
||||
|
||||
ensure_ufw_rule() {
|
||||
local port="$1"
|
||||
local comment="$2"
|
||||
if ! ssh_target "sudo ufw status numbered | grep -Fq '${port}/tcp on cni0'"; then
|
||||
ssh_target "sudo ufw allow in on cni0 from ${POD_CIDR} to any port ${port} proto tcp comment '${comment}' >/dev/null"
|
||||
fi
|
||||
}
|
||||
|
||||
ensure_ufw_rule 8899 k3s-pods-agave-rpc
|
||||
ensure_ufw_rule 8900 k3s-pods-agave-ws
|
||||
ensure_ufw_rule 10000 k3s-pods-agave-grpc
|
||||
|
||||
ssh_target "sudo systemctl status --no-pager agave-rpc-k3s.socket agave-ws-k3s.socket agave-grpc-k3s.socket | sed -n '1,80p'"
|
||||
ssh_target "sudo ss -ltnp | egrep ':(8899|8900|10000)\\b' | sed -n '1,40p'"
|
||||
@@ -7,7 +7,7 @@ Minimal canary namespace for migration baseline `R001` on `sol`.
|
||||
- Reserve a dedicated namespace for the first reconstructed trade deployment.
|
||||
- Put hard upper bounds on namespace-level CPU, memory, object count, and PVC growth before application manifests land.
|
||||
- Verify that workloads in the namespace can resolve and reach the shared `trade-infra` services for `Postgres` and `Redis`.
|
||||
- Recreate the `R001` application surface in a controlled way: `Hasura`, `trade-api`, `trade-frontend`, and the first canary `trade-ingestor` path.
|
||||
- Recreate the `R001` application surface in a controlled way: `Hasura`, `trade-api`, `trade-frontend`, the first canary `trade-ingestor` path, and the first DLOB hot-path components.
|
||||
|
||||
## Current Guardrails
|
||||
|
||||
@@ -34,16 +34,21 @@ Minimal canary namespace for migration baseline `R001` on `sol`.
|
||||
- Current shared infrastructure endpoints expected by canary workloads:
|
||||
- `postgres-host.trade-infra.svc.cluster.local:5432`
|
||||
- `redis-host.trade-infra.svc.cluster.local:6379`
|
||||
- `agave-rpc-host.trade-infra.svc.cluster.local:8899`
|
||||
- `agave-ws-host.trade-infra.svc.cluster.local:8900`
|
||||
- `agave-grpc-host.trade-infra.svc.cluster.local:10000`
|
||||
|
||||
## Application Surface
|
||||
|
||||
- `postgres` in this namespace is an `ExternalName` alias that points to `postgres-host.trade-infra.svc.cluster.local`.
|
||||
- `Hasura` uses the live `R001` secrets copied from `trade-staging`, but connects to the host `Postgres` on `sol`.
|
||||
- `trade-api` and `trade-frontend` use the current live images from Gitea registry and the same bootstrap wrapper/config pattern as the source environment.
|
||||
- `dlob-publisher-hot` now targets the host validator on `sol` through `trade-infra` services and writes `dlob-hot:*` into the shared Redis host service.
|
||||
- `dlob-hot-redis-to-postgres-raw-writer` and `dlob-hot-postgres-to-postgres-derived-writer` rebuild the first live DLOB derived path on `sol`.
|
||||
- The canary workflow re-runs:
|
||||
- `postgres-migrate`
|
||||
- `hasura-bootstrap`
|
||||
before it waits for `Hasura`, `trade-api`, `trade-frontend`, and `trade-ingestor` to become healthy.
|
||||
before it waits for `Hasura`, `trade-api`, `trade-frontend`, `trade-ingestor`, and the DLOB hot-path deployments to become healthy.
|
||||
- The current canary `trade-ingestor` is intentionally pinned to the schema already reconstructed on `sol` and reads from `dlob_stats_latest`.
|
||||
- The exact live `R001` ingestor path that reads `dlob_*_derived_latest` remains a follow-up substep after the DLOB writer chain is reconstructed.
|
||||
|
||||
@@ -52,8 +57,11 @@ Minimal canary namespace for migration baseline `R001` on `sol`.
|
||||
From the repository root:
|
||||
|
||||
```bash
|
||||
./environments/sol/trade-infra/scripts/prepare-sol-agave-access.sh
|
||||
kubectl apply -k environments/sol/trade-infra
|
||||
./environments/sol/trade-r001-canary/scripts/prepare-sol-postgres.sh
|
||||
./environments/sol/trade-r001-canary/scripts/create-gitea-registry-secret.sh
|
||||
./environments/sol/trade-r001-canary/scripts/create-trade-dlob-rpc-secret.sh
|
||||
./environments/sol/trade-r001-canary/scripts/sync-live-secrets.sh
|
||||
```
|
||||
|
||||
|
||||
@@ -98,12 +98,17 @@ async function main() {
|
||||
const source = 'default';
|
||||
|
||||
const baseTicks = { schema: 'public', name: 'drift_ticks' };
|
||||
const dlobHotSnapshotLatestTable = { schema: 'public', name: 'dlob_hot_snapshot_latest' };
|
||||
const dlobHotDerivedLatestTable = { schema: 'public', name: 'dlob_hot_derived_latest' };
|
||||
const dlobAllDerivedLatestTable = { schema: 'public', name: 'dlob_all_derived_latest' };
|
||||
const dlobL2LatestTable = { schema: 'public', name: 'dlob_l2_latest' };
|
||||
const dlobStatsLatestTable = { schema: 'public', name: 'dlob_stats_latest' };
|
||||
const dlobDepthBpsLatestTable = { schema: 'public', name: 'dlob_depth_bps_latest' };
|
||||
const dlobSlippageLatestTable = { schema: 'public', name: 'dlob_slippage_latest' };
|
||||
const dlobSlippageLatestV2Table = { schema: 'public', name: 'dlob_slippage_latest_v2' };
|
||||
const candlesCacheTable = { schema: 'public', name: 'drift_candles_cache' };
|
||||
const dlobHotDerivedTsTable = { schema: 'public', name: 'dlob_hot_derived_ts' };
|
||||
const dlobAllDerivedTsTable = { schema: 'public', name: 'dlob_all_derived_ts' };
|
||||
const dlobStatsTsTable = { schema: 'public', name: 'dlob_stats_ts' };
|
||||
const dlobDepthBpsTsTable = { schema: 'public', name: 'dlob_depth_bps_ts' };
|
||||
const dlobSlippageTsTable = { schema: 'public', name: 'dlob_slippage_ts' };
|
||||
@@ -373,6 +378,171 @@ async function main() {
|
||||
'updated_at',
|
||||
], { publicFilter: dlobPublicFilter });
|
||||
|
||||
await ensurePublicSelectTable(dlobHotSnapshotLatestTable, [
|
||||
'source',
|
||||
'redis_key',
|
||||
'snapshot_kind',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'market_name',
|
||||
'is_indicative',
|
||||
'ts_ms',
|
||||
'slot',
|
||||
'market_slot',
|
||||
'payload_hash',
|
||||
'mark_price_raw',
|
||||
'oracle_price_raw',
|
||||
'best_bid_price_raw',
|
||||
'best_ask_price_raw',
|
||||
'spread_pct_raw',
|
||||
'spread_quote_raw',
|
||||
'oracle_data',
|
||||
'bids',
|
||||
'asks',
|
||||
'payload',
|
||||
'updated_at',
|
||||
]);
|
||||
|
||||
await ensurePublicSelectTable(dlobHotDerivedLatestTable, [
|
||||
'source',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'market_name',
|
||||
'is_indicative',
|
||||
'ts_ms',
|
||||
'slot',
|
||||
'market_slot',
|
||||
'mark_price',
|
||||
'oracle_price',
|
||||
'best_bid_price',
|
||||
'best_ask_price',
|
||||
'mid_price',
|
||||
'spread_quote',
|
||||
'spread_bps',
|
||||
'depth_levels',
|
||||
'bid_levels',
|
||||
'ask_levels',
|
||||
'top_bid_size',
|
||||
'top_ask_size',
|
||||
'top_bid_notional',
|
||||
'top_ask_notional',
|
||||
'depth_bid_base',
|
||||
'depth_ask_base',
|
||||
'depth_bid_quote',
|
||||
'depth_ask_quote',
|
||||
'imbalance',
|
||||
'bids_norm',
|
||||
'asks_norm',
|
||||
'raw_payload_hash',
|
||||
'updated_at',
|
||||
]);
|
||||
|
||||
await ensurePublicSelectTable(dlobAllDerivedLatestTable, [
|
||||
'source',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'market_name',
|
||||
'is_indicative',
|
||||
'ts_ms',
|
||||
'slot',
|
||||
'market_slot',
|
||||
'mark_price',
|
||||
'oracle_price',
|
||||
'best_bid_price',
|
||||
'best_ask_price',
|
||||
'mid_price',
|
||||
'spread_quote',
|
||||
'spread_bps',
|
||||
'depth_levels',
|
||||
'bid_levels',
|
||||
'ask_levels',
|
||||
'top_bid_size',
|
||||
'top_ask_size',
|
||||
'top_bid_notional',
|
||||
'top_ask_notional',
|
||||
'depth_bid_base',
|
||||
'depth_ask_base',
|
||||
'depth_bid_quote',
|
||||
'depth_ask_quote',
|
||||
'imbalance',
|
||||
'bids_norm',
|
||||
'asks_norm',
|
||||
'raw_payload_hash',
|
||||
'updated_at',
|
||||
]);
|
||||
|
||||
await ensurePublicSelectTable(dlobHotDerivedTsTable, [
|
||||
'event_ts',
|
||||
'id',
|
||||
'source',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'market_name',
|
||||
'is_indicative',
|
||||
'ts_ms',
|
||||
'slot',
|
||||
'market_slot',
|
||||
'mark_price',
|
||||
'oracle_price',
|
||||
'best_bid_price',
|
||||
'best_ask_price',
|
||||
'mid_price',
|
||||
'spread_quote',
|
||||
'spread_bps',
|
||||
'depth_levels',
|
||||
'bid_levels',
|
||||
'ask_levels',
|
||||
'top_bid_size',
|
||||
'top_ask_size',
|
||||
'top_bid_notional',
|
||||
'top_ask_notional',
|
||||
'depth_bid_base',
|
||||
'depth_ask_base',
|
||||
'depth_bid_quote',
|
||||
'depth_ask_quote',
|
||||
'imbalance',
|
||||
'bids_norm',
|
||||
'asks_norm',
|
||||
'raw_payload_hash',
|
||||
'inserted_at',
|
||||
]);
|
||||
|
||||
await ensurePublicSelectTable(dlobAllDerivedTsTable, [
|
||||
'event_ts',
|
||||
'id',
|
||||
'source',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'market_name',
|
||||
'is_indicative',
|
||||
'ts_ms',
|
||||
'slot',
|
||||
'market_slot',
|
||||
'mark_price',
|
||||
'oracle_price',
|
||||
'best_bid_price',
|
||||
'best_ask_price',
|
||||
'mid_price',
|
||||
'spread_quote',
|
||||
'spread_bps',
|
||||
'depth_levels',
|
||||
'bid_levels',
|
||||
'ask_levels',
|
||||
'top_bid_size',
|
||||
'top_ask_size',
|
||||
'top_bid_notional',
|
||||
'top_ask_notional',
|
||||
'depth_bid_base',
|
||||
'depth_ask_base',
|
||||
'depth_bid_quote',
|
||||
'depth_ask_quote',
|
||||
'imbalance',
|
||||
'bids_norm',
|
||||
'asks_norm',
|
||||
'raw_payload_hash',
|
||||
'inserted_at',
|
||||
]);
|
||||
|
||||
await ensurePublicSelectTable(dlobStatsTsTable, [
|
||||
'ts',
|
||||
'id',
|
||||
|
||||
@@ -858,3 +858,271 @@ BEGIN
|
||||
PERFORM add_retention_policy('dlob_slippage_ts_v2', INTERVAL '7 days');
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
END $$;
|
||||
|
||||
-- Canonical raw DLOB hot snapshots written from Redis to PostgreSQL.
|
||||
CREATE TABLE IF NOT EXISTS public.dlob_hot_snapshot_latest (
|
||||
source TEXT NOT NULL,
|
||||
redis_key TEXT NOT NULL,
|
||||
snapshot_kind TEXT NOT NULL CHECK (snapshot_kind IN ('orderbook_l2', 'orderbook_l3', 'best_makers')),
|
||||
market_type TEXT NOT NULL,
|
||||
market_index INTEGER NOT NULL,
|
||||
market_name TEXT NOT NULL,
|
||||
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
ts_ms BIGINT NOT NULL,
|
||||
slot BIGINT,
|
||||
market_slot BIGINT,
|
||||
payload_hash TEXT NOT NULL,
|
||||
mark_price_raw TEXT,
|
||||
oracle_price_raw TEXT,
|
||||
best_bid_price_raw TEXT,
|
||||
best_ask_price_raw TEXT,
|
||||
spread_pct_raw TEXT,
|
||||
spread_quote_raw TEXT,
|
||||
oracle_data JSONB,
|
||||
bids JSONB,
|
||||
asks JSONB,
|
||||
payload JSONB NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (source, market_type, market_index, snapshot_kind, is_indicative)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_latest_updated_at_idx
|
||||
ON public.dlob_hot_snapshot_latest (updated_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_latest_source_market_idx
|
||||
ON public.dlob_hot_snapshot_latest (source, market_type, market_index, updated_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.dlob_hot_snapshot_ts (
|
||||
event_ts TIMESTAMPTZ NOT NULL,
|
||||
id BIGSERIAL NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
redis_key TEXT NOT NULL,
|
||||
snapshot_kind TEXT NOT NULL CHECK (snapshot_kind IN ('orderbook_l2', 'orderbook_l3', 'best_makers')),
|
||||
market_type TEXT NOT NULL,
|
||||
market_index INTEGER NOT NULL,
|
||||
market_name TEXT NOT NULL,
|
||||
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
ts_ms BIGINT NOT NULL,
|
||||
slot BIGINT,
|
||||
market_slot BIGINT,
|
||||
payload_hash TEXT NOT NULL,
|
||||
mark_price_raw TEXT,
|
||||
oracle_price_raw TEXT,
|
||||
best_bid_price_raw TEXT,
|
||||
best_ask_price_raw TEXT,
|
||||
spread_pct_raw TEXT,
|
||||
spread_quote_raw TEXT,
|
||||
oracle_data JSONB,
|
||||
bids JSONB,
|
||||
asks JSONB,
|
||||
payload JSONB NOT NULL,
|
||||
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (event_ts, id)
|
||||
);
|
||||
|
||||
SELECT create_hypertable('dlob_hot_snapshot_ts', 'event_ts', if_not_exists => TRUE, migrate_data => TRUE);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS dlob_hot_snapshot_ts_dedupe_idx
|
||||
ON public.dlob_hot_snapshot_ts (event_ts, source, redis_key, payload_hash);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_ts_market_ts_desc_idx
|
||||
ON public.dlob_hot_snapshot_ts (market_type, market_index, event_ts DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_ts_source_market_ts_desc_idx
|
||||
ON public.dlob_hot_snapshot_ts (source, market_type, market_index, event_ts DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.dlob_hot_derived_latest (
|
||||
source TEXT NOT NULL,
|
||||
market_type TEXT NOT NULL,
|
||||
market_index INTEGER NOT NULL,
|
||||
market_name TEXT NOT NULL,
|
||||
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
ts_ms BIGINT NOT NULL,
|
||||
slot BIGINT,
|
||||
market_slot BIGINT,
|
||||
mark_price NUMERIC,
|
||||
oracle_price NUMERIC,
|
||||
best_bid_price NUMERIC,
|
||||
best_ask_price NUMERIC,
|
||||
mid_price NUMERIC,
|
||||
spread_quote NUMERIC,
|
||||
spread_bps NUMERIC,
|
||||
depth_levels INTEGER NOT NULL,
|
||||
bid_levels INTEGER NOT NULL,
|
||||
ask_levels INTEGER NOT NULL,
|
||||
top_bid_size NUMERIC,
|
||||
top_ask_size NUMERIC,
|
||||
top_bid_notional NUMERIC,
|
||||
top_ask_notional NUMERIC,
|
||||
depth_bid_base NUMERIC NOT NULL DEFAULT 0,
|
||||
depth_ask_base NUMERIC NOT NULL DEFAULT 0,
|
||||
depth_bid_quote NUMERIC NOT NULL DEFAULT 0,
|
||||
depth_ask_quote NUMERIC NOT NULL DEFAULT 0,
|
||||
imbalance NUMERIC,
|
||||
bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
raw_payload_hash TEXT NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (source, market_type, market_index, is_indicative)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dlob_hot_derived_latest_updated_at_idx
|
||||
ON public.dlob_hot_derived_latest (updated_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dlob_hot_derived_latest_source_market_idx
|
||||
ON public.dlob_hot_derived_latest (source, market_type, market_index, updated_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.dlob_hot_derived_ts (
|
||||
event_ts TIMESTAMPTZ NOT NULL,
|
||||
id BIGSERIAL NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
market_type TEXT NOT NULL,
|
||||
market_index INTEGER NOT NULL,
|
||||
market_name TEXT NOT NULL,
|
||||
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
ts_ms BIGINT NOT NULL,
|
||||
slot BIGINT,
|
||||
market_slot BIGINT,
|
||||
mark_price NUMERIC,
|
||||
oracle_price NUMERIC,
|
||||
best_bid_price NUMERIC,
|
||||
best_ask_price NUMERIC,
|
||||
mid_price NUMERIC,
|
||||
spread_quote NUMERIC,
|
||||
spread_bps NUMERIC,
|
||||
depth_levels INTEGER NOT NULL,
|
||||
bid_levels INTEGER NOT NULL,
|
||||
ask_levels INTEGER NOT NULL,
|
||||
top_bid_size NUMERIC,
|
||||
top_ask_size NUMERIC,
|
||||
top_bid_notional NUMERIC,
|
||||
top_ask_notional NUMERIC,
|
||||
depth_bid_base NUMERIC NOT NULL DEFAULT 0,
|
||||
depth_ask_base NUMERIC NOT NULL DEFAULT 0,
|
||||
depth_bid_quote NUMERIC NOT NULL DEFAULT 0,
|
||||
depth_ask_quote NUMERIC NOT NULL DEFAULT 0,
|
||||
imbalance NUMERIC,
|
||||
bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
raw_payload_hash TEXT NOT NULL,
|
||||
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (event_ts, id)
|
||||
);
|
||||
|
||||
SELECT create_hypertable('dlob_hot_derived_ts', 'event_ts', if_not_exists => TRUE, migrate_data => TRUE);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS dlob_hot_derived_ts_dedupe_idx
|
||||
ON public.dlob_hot_derived_ts (event_ts, source, market_type, market_index, is_indicative, raw_payload_hash);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dlob_hot_derived_ts_market_ts_desc_idx
|
||||
ON public.dlob_hot_derived_ts (market_type, market_index, event_ts DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dlob_hot_derived_ts_source_market_ts_desc_idx
|
||||
ON public.dlob_hot_derived_ts (source, market_type, market_index, event_ts DESC);
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
PERFORM add_retention_policy('dlob_hot_snapshot_ts', INTERVAL '30 days');
|
||||
EXCEPTION WHEN duplicate_object THEN NULL;
|
||||
END $$;
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
PERFORM add_retention_policy('dlob_hot_derived_ts', INTERVAL '180 days');
|
||||
EXCEPTION WHEN duplicate_object THEN NULL;
|
||||
END $$;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.dlob_all_derived_latest (
|
||||
source TEXT NOT NULL,
|
||||
market_type TEXT NOT NULL,
|
||||
market_index INTEGER NOT NULL,
|
||||
market_name TEXT NOT NULL,
|
||||
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
ts_ms BIGINT NOT NULL,
|
||||
slot BIGINT,
|
||||
market_slot BIGINT,
|
||||
mark_price NUMERIC,
|
||||
oracle_price NUMERIC,
|
||||
best_bid_price NUMERIC,
|
||||
best_ask_price NUMERIC,
|
||||
mid_price NUMERIC,
|
||||
spread_quote NUMERIC,
|
||||
spread_bps NUMERIC,
|
||||
depth_levels INTEGER NOT NULL,
|
||||
bid_levels INTEGER NOT NULL,
|
||||
ask_levels INTEGER NOT NULL,
|
||||
top_bid_size NUMERIC,
|
||||
top_ask_size NUMERIC,
|
||||
top_bid_notional NUMERIC,
|
||||
top_ask_notional NUMERIC,
|
||||
depth_bid_base NUMERIC NOT NULL DEFAULT 0,
|
||||
depth_ask_base NUMERIC NOT NULL DEFAULT 0,
|
||||
depth_bid_quote NUMERIC NOT NULL DEFAULT 0,
|
||||
depth_ask_quote NUMERIC NOT NULL DEFAULT 0,
|
||||
imbalance NUMERIC,
|
||||
bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
raw_payload_hash TEXT NOT NULL,
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (source, market_type, market_index, is_indicative)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dlob_all_derived_latest_updated_at_idx
|
||||
ON public.dlob_all_derived_latest (updated_at DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dlob_all_derived_latest_source_market_idx
|
||||
ON public.dlob_all_derived_latest (source, market_type, market_index, updated_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS public.dlob_all_derived_ts (
|
||||
event_ts TIMESTAMPTZ NOT NULL,
|
||||
id BIGSERIAL NOT NULL,
|
||||
source TEXT NOT NULL,
|
||||
market_type TEXT NOT NULL,
|
||||
market_index INTEGER NOT NULL,
|
||||
market_name TEXT NOT NULL,
|
||||
is_indicative BOOLEAN NOT NULL DEFAULT FALSE,
|
||||
ts_ms BIGINT NOT NULL,
|
||||
slot BIGINT,
|
||||
market_slot BIGINT,
|
||||
mark_price NUMERIC,
|
||||
oracle_price NUMERIC,
|
||||
best_bid_price NUMERIC,
|
||||
best_ask_price NUMERIC,
|
||||
mid_price NUMERIC,
|
||||
spread_quote NUMERIC,
|
||||
spread_bps NUMERIC,
|
||||
depth_levels INTEGER NOT NULL,
|
||||
bid_levels INTEGER NOT NULL,
|
||||
ask_levels INTEGER NOT NULL,
|
||||
top_bid_size NUMERIC,
|
||||
top_ask_size NUMERIC,
|
||||
top_bid_notional NUMERIC,
|
||||
top_ask_notional NUMERIC,
|
||||
depth_bid_base NUMERIC NOT NULL DEFAULT 0,
|
||||
depth_ask_base NUMERIC NOT NULL DEFAULT 0,
|
||||
depth_bid_quote NUMERIC NOT NULL DEFAULT 0,
|
||||
depth_ask_quote NUMERIC NOT NULL DEFAULT 0,
|
||||
imbalance NUMERIC,
|
||||
bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb,
|
||||
raw_payload_hash TEXT NOT NULL,
|
||||
inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (event_ts, id)
|
||||
);
|
||||
|
||||
SELECT create_hypertable('dlob_all_derived_ts', 'event_ts', if_not_exists => TRUE, migrate_data => TRUE);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS dlob_all_derived_ts_dedupe_idx
|
||||
ON public.dlob_all_derived_ts (event_ts, source, market_type, market_index, is_indicative, raw_payload_hash);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dlob_all_derived_ts_market_ts_desc_idx
|
||||
ON public.dlob_all_derived_ts (market_type, market_index, event_ts DESC);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS dlob_all_derived_ts_source_market_ts_desc_idx
|
||||
ON public.dlob_all_derived_ts (source, market_type, market_index, event_ts DESC);
|
||||
|
||||
DO $$
|
||||
BEGIN
|
||||
PERFORM add_retention_policy('dlob_all_derived_ts', INTERVAL '30 days');
|
||||
EXCEPTION WHEN duplicate_object THEN NULL;
|
||||
END $$;
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-hot-postgres-to-postgres-derived-writer
|
||||
namespace: trade-r001-canary
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: dlob-hot-postgres-to-postgres-derived-writer
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: dlob-hot-postgres-to-postgres-derived-writer
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: gitea-registry
|
||||
containers:
|
||||
- name: writer
|
||||
image: gitea.mpabi.pl/trade/trade-dlob-server:hot-pg-events-20260320-205517
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- node
|
||||
- /lib/scripts/dlobHotPostgresToPostgresDerivedWriter.js
|
||||
env:
|
||||
- name: DLOB_SOURCE
|
||||
value: mevnode_bot_hot_derived
|
||||
- name: RAW_SOURCE
|
||||
value: mevnode_bot_hot_raw
|
||||
- name: DLOB_POLL_MS
|
||||
value: "5000"
|
||||
- name: NORMALIZED_DEPTH
|
||||
value: "10"
|
||||
- name: PRICE_PRECISION
|
||||
value: "1000000"
|
||||
- name: BASE_PRECISION
|
||||
value: "1000000000"
|
||||
- name: PGHOST
|
||||
value: postgres
|
||||
- name: PGPORT
|
||||
value: "5432"
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_USER
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: PGDATABASE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_DB
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
@@ -0,0 +1,61 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-hot-redis-to-postgres-raw-writer
|
||||
namespace: trade-r001-canary
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: dlob-hot-redis-to-postgres-raw-writer
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: dlob-hot-redis-to-postgres-raw-writer
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: gitea-registry
|
||||
containers:
|
||||
- name: writer
|
||||
image: gitea.mpabi.pl/trade/trade-dlob-server:hot-pg-events-20260320-205517
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- node
|
||||
- /lib/scripts/dlobHotRedisToPostgresRawWriter.js
|
||||
env:
|
||||
- name: DLOB_SOURCE
|
||||
value: mevnode_bot_hot_raw
|
||||
- name: REDIS_HOST
|
||||
value: dlob-redis
|
||||
- name: REDIS_PORT
|
||||
value: "6379"
|
||||
- name: REDIS_KEY_PREFIX
|
||||
value: "dlob-hot:"
|
||||
- name: DLOB_POLL_MS
|
||||
value: "5000"
|
||||
- name: PGHOST
|
||||
value: postgres
|
||||
- name: PGPORT
|
||||
value: "5432"
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_USER
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: PGDATABASE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_DB
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
@@ -0,0 +1,130 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-publisher-hot
|
||||
namespace: trade-r001-canary
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: dlob-publisher-hot
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: dlob-publisher-hot
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: gitea-registry
|
||||
containers:
|
||||
- name: publisher
|
||||
image: gitea.mpabi.pl/trade/trade-dlob-server:grpc-teardown-fix-20260402-113736
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- node
|
||||
- /lib/publishers/dlobPublisher.js
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
env:
|
||||
- name: FETCH_CONNECT_TIMEOUT_MS
|
||||
value: "15000"
|
||||
- name: FETCH_HEADERS_TIMEOUT_MS
|
||||
value: "300000"
|
||||
- name: FETCH_BODY_TIMEOUT_MS
|
||||
value: "300000"
|
||||
- name: ENABLE_PERSISTENT_STORE
|
||||
value: "true"
|
||||
- name: DLOB_SOURCE
|
||||
value: mevnode_bot_hot
|
||||
- name: PRICE_PRECISION
|
||||
value: "1000000"
|
||||
- name: BASE_PRECISION
|
||||
value: "1000000000"
|
||||
- name: PERSISTENT_STATS_DEPTH
|
||||
value: "10"
|
||||
- name: PGHOST
|
||||
value: postgres
|
||||
- name: PGPORT
|
||||
value: "5432"
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_USER
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: PGDATABASE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_DB
|
||||
- name: PERP_MARKETS_TO_LOAD
|
||||
value: "0,20,72"
|
||||
- name: USE_GRPC
|
||||
value: "true"
|
||||
- name: USE_WEBSOCKET
|
||||
value: "true"
|
||||
- name: DISABLE_GPA_REFRESH
|
||||
value: "true"
|
||||
- name: GRPC_CLIENT
|
||||
value: yellowstone
|
||||
- name: GRPC_ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-dlob-rpc
|
||||
key: GRPC_ENDPOINT
|
||||
- name: TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-dlob-rpc
|
||||
key: TOKEN
|
||||
- name: RUNNING_LOCAL
|
||||
value: "true"
|
||||
- name: LOCAL_CACHE
|
||||
value: "true"
|
||||
- name: ENV
|
||||
value: mainnet-beta
|
||||
- name: USE_ORDER_SUBSCRIBER
|
||||
value: "true"
|
||||
- name: ELASTICACHE_HOST
|
||||
value: dlob-redis
|
||||
- name: ELASTICACHE_PORT
|
||||
value: "6379"
|
||||
- name: REDIS_CLIENT
|
||||
value: DLOB_HOT
|
||||
- name: ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-dlob-rpc
|
||||
key: ENDPOINT
|
||||
- name: WS_ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-dlob-rpc
|
||||
key: WS_ENDPOINT
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: "1"
|
||||
memory: 1Gi
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /startup
|
||||
port: http
|
||||
initialDelaySeconds: 120
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 30
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 240
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 10
|
||||
@@ -0,0 +1,12 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: dlob-redis
|
||||
namespace: trade-r001-canary
|
||||
spec:
|
||||
type: ExternalName
|
||||
externalName: redis-host.trade-infra.svc.cluster.local
|
||||
ports:
|
||||
- name: redis
|
||||
port: 6379
|
||||
targetPort: 6379
|
||||
@@ -15,6 +15,13 @@ spec:
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: trade-postgres
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
@@ -38,6 +45,13 @@ spec:
|
||||
value: drift_ticks
|
||||
- name: CANDLES_FUNCTION
|
||||
value: get_drift_candles
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
command:
|
||||
- node
|
||||
- /app/hasura-bootstrap.mjs
|
||||
|
||||
@@ -11,6 +11,7 @@ resources:
|
||||
- resourcequota.yaml
|
||||
- limitrange.yaml
|
||||
- postgres-alias-service.yaml
|
||||
- dlob-redis-alias-service.yaml
|
||||
- hasura-service.yaml
|
||||
- hasura-deployment.yaml
|
||||
- postgres-migrate-job.yaml
|
||||
@@ -20,6 +21,9 @@ resources:
|
||||
- trade-frontend-service.yaml
|
||||
- trade-frontend-deployment.yaml
|
||||
- trade-ingestor-deployment.yaml
|
||||
- dlob-publisher-hot-deployment.yaml
|
||||
- dlob-hot-redis-to-postgres-raw-writer-deployment.yaml
|
||||
- dlob-hot-postgres-to-postgres-derived-writer-deployment.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: postgres-initdb
|
||||
|
||||
@@ -15,6 +15,13 @@ spec:
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: trade-postgres
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
limits:
|
||||
cpu: 250m
|
||||
memory: 256Mi
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
|
||||
25
environments/sol/trade-r001-canary/scripts/create-trade-dlob-rpc-secret.sh
Executable file
25
environments/sol/trade-r001-canary/scripts/create-trade-dlob-rpc-secret.sh
Executable file
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
TARGET_HOST="${TARGET_HOST:-mevnode}"
|
||||
TARGET_NAMESPACE="${TARGET_NAMESPACE:-trade-r001-canary}"
|
||||
|
||||
RPC_URL="${RPC_URL:-http://agave-rpc-host.trade-infra.svc.cluster.local:8899}"
|
||||
WS_URL="${WS_URL:-ws://agave-ws-host.trade-infra.svc.cluster.local:8900}"
|
||||
GRPC_URL="${GRPC_URL:-http://agave-grpc-host.trade-infra.svc.cluster.local:10000}"
|
||||
|
||||
ssh_target() {
|
||||
ssh -o StrictHostKeyChecking=no "$TARGET_HOST" "$@"
|
||||
}
|
||||
|
||||
TOKEN="$(ssh_target 'sudo cat /etc/agave/geyser.x_token')"
|
||||
|
||||
ssh_target "sudo k3s kubectl get ns ${TARGET_NAMESPACE} >/dev/null 2>&1 || sudo k3s kubectl create ns ${TARGET_NAMESPACE} >/dev/null"
|
||||
ssh_target "sudo k3s kubectl -n ${TARGET_NAMESPACE} create secret generic trade-dlob-rpc \
|
||||
--from-literal=ENDPOINT='${RPC_URL}' \
|
||||
--from-literal=WS_ENDPOINT='${WS_URL}' \
|
||||
--from-literal=GRPC_ENDPOINT='${GRPC_URL}' \
|
||||
--from-literal=TOKEN='${TOKEN}' \
|
||||
--dry-run=client -o yaml | sudo k3s kubectl apply -f - >/dev/null"
|
||||
|
||||
echo "Created trade-dlob-rpc in ${TARGET_NAMESPACE}"
|
||||
Reference in New Issue
Block a user