feat(sol): add canary dlob all path

.gitea/workflows/deploy-trade-r001-canary.yaml

@@ -5,6 +5,7 @@ on:
     branches:
       - main
     paths:
+      - environments/sol/trade-infra/**
       - environments/sol/trade-r001-canary/**
       - .gitea/workflows/deploy-trade-r001-canary.yaml
   workflow_dispatch:
@@ -67,8 +68,10 @@ jobs:
             deploy/trade-frontend \
             deploy/trade-ingestor \
             deploy/dlob-publisher-hot \
+            deploy/dlob-publisher-all \
             deploy/dlob-hot-redis-to-postgres-raw-writer \
-            deploy/dlob-hot-postgres-to-postgres-derived-writer
+            deploy/dlob-hot-postgres-to-postgres-derived-writer \
+            deploy/dlob-all-redis-to-postgres-derived-writer
 
       - name: Wait for database and metadata bootstrap
         env:
@@ -86,8 +89,10 @@ jobs:
           kubectl -n trade-r001-canary rollout status deploy/trade-frontend --timeout=300s
           kubectl -n trade-r001-canary rollout status deploy/trade-ingestor --timeout=300s
           kubectl -n trade-r001-canary rollout status deploy/dlob-publisher-hot --timeout=420s
+          kubectl -n trade-r001-canary rollout status deploy/dlob-publisher-all --timeout=420s
           kubectl -n trade-r001-canary rollout status deploy/dlob-hot-redis-to-postgres-raw-writer --timeout=300s
           kubectl -n trade-r001-canary rollout status deploy/dlob-hot-postgres-to-postgres-derived-writer --timeout=300s
+          kubectl -n trade-r001-canary rollout status deploy/dlob-all-redis-to-postgres-derived-writer --timeout=300s
           kubectl -n trade-r001-canary get deploy,pods -o wide
 
       - name: Verify trade-ingestor runtime
@@ -146,8 +151,10 @@ jobs:
           KUBECONFIG: /tmp/kubeconfig
         run: |
           kubectl -n trade-r001-canary logs deploy/dlob-publisher-hot --tail=20
+          kubectl -n trade-r001-canary logs deploy/dlob-publisher-all --tail=20
           kubectl -n trade-r001-canary logs deploy/dlob-hot-redis-to-postgres-raw-writer --tail=20
           kubectl -n trade-r001-canary logs deploy/dlob-hot-postgres-to-postgres-derived-writer --tail=20
+          kubectl -n trade-r001-canary logs deploy/dlob-all-redis-to-postgres-derived-writer --tail=20
           pod_name="$(kubectl -n trade-r001-canary get pod -l app.kubernetes.io/name=trade-ingestor -o jsonpath='{.items[0].metadata.name}')"
           kubectl -n trade-r001-canary exec -i "$pod_name" -- node - <<'JS'
           const endpoint = 'http://hasura:8080/v1/graphql';
@@ -159,6 +166,11 @@ jobs:
                 updated_at
                 source
               }
+              dlob_all_derived_latest(limit: 1, order_by: { updated_at: desc }) {
+                market_name
+                updated_at
+                source
+              }
             }
           `;
 
@@ -173,11 +185,15 @@ jobs:
               signal: AbortSignal.timeout(10000),
             });
             const payload = await response.json();
-            const rows = payload?.data?.dlob_hot_derived_latest || [];
+            const hotRows = payload?.data?.dlob_hot_derived_latest || [];
-            if (!rows.length) {
+            const allRows = payload?.data?.dlob_all_derived_latest || [];
+            if (!hotRows.length) {
               throw new Error('No rows in dlob_hot_derived_latest yet');
             }
-            console.log(JSON.stringify(rows[0], null, 2));
+            if (!allRows.length) {
+              throw new Error('No rows in dlob_all_derived_latest yet');
+            }
+            console.log(JSON.stringify({ hot: hotRows[0], all: allRows[0] }, null, 2));
           }
 
           (async () => {

environments/sol/trade-r001-canary/README.md

@@ -44,11 +44,13 @@ Minimal canary namespace for migration baseline `R001` on `sol`.
 - `Hasura` uses the live `R001` secrets copied from `trade-staging`, but connects to the host `Postgres` on `sol`.
 - `trade-api` and `trade-frontend` use the current live images from Gitea registry and the same bootstrap wrapper/config pattern as the source environment.
 - `dlob-publisher-hot` now targets the host validator on `sol` through `trade-infra` services and writes `dlob-hot:*` into the shared Redis host service.
+- `dlob-publisher-all` now targets the same host validator path on `sol` and writes `dlob-all:*` into the shared Redis host service.
 - `dlob-hot-redis-to-postgres-raw-writer` and `dlob-hot-postgres-to-postgres-derived-writer` rebuild the first live DLOB derived path on `sol`.
+- `dlob-all-redis-to-postgres-derived-writer` rebuilds the live full-market derived DLOB path on `sol`.
 - The canary workflow re-runs:
   - `postgres-migrate`
   - `hasura-bootstrap`
-  before it waits for `Hasura`, `trade-api`, `trade-frontend`, `trade-ingestor`, and the DLOB hot-path deployments to become healthy.
+  before it waits for `Hasura`, `trade-api`, `trade-frontend`, `trade-ingestor`, and the DLOB hot/all-path deployments to become healthy.
 - The current canary `trade-ingestor` is intentionally pinned to the schema already reconstructed on `sol` and reads from `dlob_stats_latest`.
 - The exact live `R001` ingestor path that reads `dlob_*_derived_latest` remains a follow-up substep after the DLOB writer chain is reconstructed.
 

environments/sol/trade-r001-canary/dlob-all-redis-to-postgres-derived-writer-deployment.yaml

@@ -0,0 +1,72 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dlob-all-redis-to-postgres-derived-writer
+  namespace: trade-r001-canary
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: dlob-all-redis-to-postgres-derived-writer
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: dlob-all-redis-to-postgres-derived-writer
+    spec:
+      imagePullSecrets:
+        - name: gitea-registry
+      containers:
+        - name: writer
+          image: gitea.mpabi.pl/trade/trade-dlob-server:all-derived-20260317-014016
+          imagePullPolicy: IfNotPresent
+          command:
+            - node
+            - /lib/scripts/dlobAllRedisToPostgresDerivedWriter.js
+          env:
+            - name: DLOB_SOURCE
+              value: mevnode_bot_all_derived
+            - name: REDIS_HOST
+              value: dlob-redis
+            - name: REDIS_PORT
+              value: "6379"
+            - name: REDIS_KEY_PREFIX
+              value: "dlob-all:"
+            - name: DLOB_POLL_MS
+              value: "1000"
+            - name: NORMALIZED_DEPTH
+              value: "10"
+            - name: PRICE_PRECISION
+              value: "1000000"
+            - name: BASE_PRECISION
+              value: "1000000000"
+            - name: PGHOST
+              value: postgres
+            - name: PGPORT
+              value: "5432"
+            - name: PGUSER
+              valueFrom:
+                secretKeyRef:
+                  name: trade-postgres
+                  key: POSTGRES_USER
+            - name: PGPASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: trade-postgres
+                  key: POSTGRES_PASSWORD
+            - name: PGDATABASE
+              valueFrom:
+                secretKeyRef:
+                  name: trade-postgres
+                  key: POSTGRES_DB
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: 500m
+              memory: 2Gi

environments/sol/trade-r001-canary/dlob-publisher-all-deployment.yaml

@@ -0,0 +1,116 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dlob-publisher-all
+  namespace: trade-r001-canary
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: dlob-publisher-all
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: dlob-publisher-all
+    spec:
+      imagePullSecrets:
+        - name: gitea-registry
+      containers:
+        - name: publisher
+          image: gitea.mpabi.pl/trade/trade-dlob-server:grpc-teardown-fix-20260402-113736
+          imagePullPolicy: IfNotPresent
+          command:
+            - node
+            - /lib/publishers/dlobPublisher.js
+          ports:
+            - name: http
+              containerPort: 8080
+          env:
+            - name: FETCH_CONNECT_TIMEOUT_MS
+              value: "15000"
+            - name: FETCH_HEADERS_TIMEOUT_MS
+              value: "300000"
+            - name: FETCH_BODY_TIMEOUT_MS
+              value: "300000"
+            - name: RUNNING_LOCAL
+              value: "true"
+            - name: LOCAL_CACHE
+              value: "true"
+            - name: ENV
+              value: mainnet-beta
+            - name: USE_ORDER_SUBSCRIBER
+              value: "true"
+            - name: USE_GRPC
+              value: "true"
+            - name: USE_WEBSOCKET
+              value: "true"
+            - name: DISABLE_GPA_REFRESH
+              value: "true"
+            - name: ENABLE_TOB_MONITORING
+              value: "false"
+            - name: ELASTICACHE_HOST
+              value: dlob-redis
+            - name: ELASTICACHE_PORT
+              value: "6379"
+            - name: REDIS_CLIENT
+              value: DLOB_ALL
+            - name: GRPC_CLIENT
+              value: yellowstone
+            - name: ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: trade-dlob-rpc
+                  key: ENDPOINT
+            - name: WS_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: trade-dlob-rpc
+                  key: WS_ENDPOINT
+            - name: GRPC_ENDPOINT
+              valueFrom:
+                secretKeyRef:
+                  name: trade-dlob-rpc
+                  key: GRPC_ENDPOINT
+            - name: TOKEN
+              valueFrom:
+                secretKeyRef:
+                  name: trade-dlob-rpc
+                  key: TOKEN
+            - name: KILLSWITCH_SLOT_DIFF_THRESHOLD
+              value: "1000000000"
+          resources:
+            requests:
+              cpu: 250m
+              memory: 512Mi
+            limits:
+              cpu: 500m
+              memory: 1Gi
+          readinessProbe:
+            httpGet:
+              path: /startup
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 24
+          startupProbe:
+            httpGet:
+              path: /startup
+              port: http
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            timeoutSeconds: 3
+            failureThreshold: 180
+          livenessProbe:
+            httpGet:
+              path: /health
+              port: http
+            initialDelaySeconds: 30
+            periodSeconds: 20
+            timeoutSeconds: 3
+            failureThreshold: 10

environments/sol/trade-r001-canary/kustomization.yaml

@@ -22,8 +22,10 @@ resources:
   - trade-frontend-deployment.yaml
   - trade-ingestor-deployment.yaml
   - dlob-publisher-hot-deployment.yaml
+  - dlob-publisher-all-deployment.yaml
   - dlob-hot-redis-to-postgres-raw-writer-deployment.yaml
   - dlob-hot-postgres-to-postgres-derived-writer-deployment.yaml
+  - dlob-all-redis-to-postgres-derived-writer-deployment.yaml
 
 configMapGenerator:
   - name: postgres-initdb