feat(dlob): pin markets and wire mevnode endpoints

- Limit DLOB workers/ingestors to SOL-PERP, DOGE-PERP, JUP-PERP across base and staging config. - Set publisher market ids to [0,7,24] for drift protocol. - Add overlay patches for dlob-publisher and dlob-server to use wg0 RPC endpoints 10.66.66.1:8899/8900 in staging and prod. - Extend Agave dashboard and add PrometheusRules for RPC up/lag/I/O alerts. - Ensure overlays reference new patches for automated ArgoCD rollouts.
chore(grafana): set default time range to now
2026-02-15 00:40:50 +01:00 · 2026-02-14 14:08:11 +01:00
24 changed files with 199 additions and 16 deletions
--- a/kustomize/base/dlob-depth-worker/deployment-drift.yaml
+++ b/kustomize/base/dlob-depth-worker/deployment-drift.yaml
@@ -29,7 +29,7 @@ spec:
            - name: DLOB_SOURCE
              value: drift
            - name: DLOB_MARKETS
-              value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
+              value: SOL-PERP,DOGE-PERP,JUP-PERP
            - name: DLOB_POLL_MS
              value: "1000"
            - name: DLOB_DEPTH_BPS_BANDS
--- a/kustomize/base/dlob-depth-worker/deployment.yaml
+++ b/kustomize/base/dlob-depth-worker/deployment.yaml
@@ -29,7 +29,7 @@ spec:
            - name: DLOB_SOURCE
              value: mevnode
            - name: DLOB_MARKETS
-              value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
+              value: SOL-PERP,DOGE-PERP,JUP-PERP
            - name: DLOB_POLL_MS
              value: "1000"
            - name: DLOB_DEPTH_BPS_BANDS
--- a/kustomize/base/dlob-depth-worker/worker.mjs
+++ b/kustomize/base/dlob-depth-worker/worker.mjs
@@ -65,7 +65,7 @@ function resolveConfig() {
  const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined;

  const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode';
-  const markets = envList('DLOB_MARKETS', 'PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP');
+  const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
  const pollMs = clampInt(process.env.DLOB_POLL_MS, 250, 60_000, 1000);
  const bandsBps = envIntList('DLOB_DEPTH_BPS_BANDS', '5,10,20,50,100,200');

--- a/kustomize/base/dlob-slippage-worker/deployment-drift.yaml
+++ b/kustomize/base/dlob-slippage-worker/deployment-drift.yaml
@@ -29,7 +29,7 @@ spec:
            - name: DLOB_SOURCE
              value: drift
            - name: DLOB_MARKETS
-              value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
+              value: SOL-PERP,DOGE-PERP,JUP-PERP
            - name: DLOB_POLL_MS
              value: "1000"
            - name: DLOB_SLIPPAGE_SIZES_USD
--- a/kustomize/base/dlob-slippage-worker/deployment.yaml
+++ b/kustomize/base/dlob-slippage-worker/deployment.yaml
@@ -29,7 +29,7 @@ spec:
            - name: DLOB_SOURCE
              value: mevnode
            - name: DLOB_MARKETS
-              value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
+              value: SOL-PERP,DOGE-PERP,JUP-PERP
            - name: DLOB_POLL_MS
              value: "1000"
            - name: DLOB_SLIPPAGE_SIZES_USD
--- a/kustomize/base/dlob-slippage-worker/worker.mjs
+++ b/kustomize/base/dlob-slippage-worker/worker.mjs
@@ -56,7 +56,7 @@ function resolveConfig() {
  const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined;

  const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode';
-  const markets = envList('DLOB_MARKETS', 'PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP');
+  const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
  const pollMs = clampInt(process.env.DLOB_POLL_MS, 250, 60_000, 1000);

  const sizesUsd = envList('DLOB_SLIPPAGE_SIZES_USD', '10,25,50,100,250,500,1000')
--- a/kustomize/base/dlob-ts-archiver/deployment-drift.yaml
+++ b/kustomize/base/dlob-ts-archiver/deployment-drift.yaml
@@ -29,7 +29,7 @@ spec:
            - name: DLOB_SOURCE
              value: drift
            - name: DLOB_MARKETS
-              value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
+              value: SOL-PERP,DOGE-PERP,JUP-PERP
            - name: DLOB_TS_POLL_MS
              value: "1000"
          command: ["node", "/app/worker.mjs"]
--- a/kustomize/base/dlob-ts-archiver/deployment.yaml
+++ b/kustomize/base/dlob-ts-archiver/deployment.yaml
@@ -29,7 +29,7 @@ spec:
            - name: DLOB_SOURCE
              value: mevnode
            - name: DLOB_MARKETS
-              value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
+              value: SOL-PERP,DOGE-PERP,JUP-PERP
            - name: DLOB_TS_POLL_MS
              value: "1000"
          command: ["node", "/app/worker.mjs"]
--- a/kustomize/base/dlob-ts-archiver/worker.mjs
+++ b/kustomize/base/dlob-ts-archiver/worker.mjs
@@ -50,7 +50,7 @@ function resolveConfig() {
  const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined;

  const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode';
-  const markets = envList('DLOB_MARKETS', 'PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP');
+  const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
  const pollMs = clampInt(process.env.DLOB_TS_POLL_MS, 500, 60_000, 1000);

  return { hasuraUrl, hasuraAdminSecret, hasuraAuthToken, dlobSource, markets, pollMs };
--- a/kustomize/base/dlob-worker/deployment-drift.yaml
+++ b/kustomize/base/dlob-worker/deployment-drift.yaml
@@ -35,7 +35,7 @@ spec:
            - name: DLOB_FORCE_IPV6
              value: "true"
            - name: DLOB_MARKETS
-              value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
+              value: SOL-PERP,DOGE-PERP,JUP-PERP
            - name: DLOB_POLL_MS
              value: "500"
            - name: DLOB_DEPTH
--- a/kustomize/base/dlob-worker/deployment.yaml
+++ b/kustomize/base/dlob-worker/deployment.yaml
@@ -31,7 +31,7 @@ spec:
            - name: DLOB_HTTP_URL
              value: http://dlob-server:6969
            - name: DLOB_MARKETS
-              value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
+              value: SOL-PERP,DOGE-PERP,JUP-PERP
            - name: DLOB_POLL_MS
              value: "500"
            - name: DLOB_DEPTH
--- a/kustomize/base/dlob-worker/worker.mjs
+++ b/kustomize/base/dlob-worker/worker.mjs
@@ -66,7 +66,7 @@ function resolveConfig() {
  const dlobForceIpv6 = envBool('DLOB_FORCE_IPV6', false);
  const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode';

-  const markets = envList('DLOB_MARKETS', 'PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP');
+  const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
  const depth = clampInt(process.env.DLOB_DEPTH, 1, 50, 10);
  const pollMs = clampInt(process.env.DLOB_POLL_MS, 100, 10_000, 500);

--- a/kustomize/base/dlob/publisher-deployment.yaml
+++ b/kustomize/base/dlob/publisher-deployment.yaml
@@ -43,7 +43,7 @@ spec:
            - name: REDIS_CLIENT
              value: DLOB
            - name: PERP_MARKETS_TO_LOAD
-              value: "0,1,2,4,75"
+              value: "0,7,24"
            - name: ENDPOINT
              valueFrom:
                secretKeyRef:
--- a/kustomize/infra/monitoring-extras/dashboard-agave-status.yaml
+++ b/kustomize/infra/monitoring-extras/dashboard-agave-status.yaml
@@ -13,6 +13,11 @@ data:
      "timezone": "browser",
      "schemaVersion": 39,
      "version": 2,
+      "time": { "from": "now-6h", "to": "now" },
+      "timepicker": {
+        "refresh_intervals": ["5s", "10s", "30s", "1m", "5m"],
+        "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d"]
+      },
      "refresh": "10s",
      "tags": ["agave", "solana", "mpabi"],
      "templating": {
@@ -171,7 +176,63 @@ data:
          "datasource": { "type": "prometheus", "uid": "prometheus" },
          "targets": [{ "refId": "Q", "expr": "sum by (reason) (increase(grpc_client_disconnects_total{job=\"mpabi-yellowstone-geyser\"}[15m]))", "legendFormat": "{{reason}}" }],
          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 46 }
+        },
+        {
+          "id": 16,
+          "type": "stat",
+          "title": "RPC Slot Lag (slots)",
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [{ "refId": "R", "expr": "solana_rpc_slot_lag{job=\"mpabi-node-exporter\",instance=\"$instance\"}" }],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "short",
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null },
+                  { "color": "yellow", "value": 20 },
+                  { "color": "red", "value": 50 }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 6, "w": 12, "x": 0, "y": 54 }
+        },
+        {
+          "id": 17,
+          "type": "stat",
+          "title": "RPC Slot Lag (szac. minuty)",
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [{ "refId": "S", "expr": "solana_rpc_slot_lag{job=\"mpabi-node-exporter\",instance=\"$instance\"} * 0.4 / 60" }],
+          "fieldConfig": {
+            "defaults": {
+              "unit": "min",
+              "decimals": 2,
+              "thresholds": {
+                "mode": "absolute",
+                "steps": [
+                  { "color": "green", "value": null },
+                  { "color": "yellow", "value": 1 },
+                  { "color": "red", "value": 2 }
+                ]
+              }
+            },
+            "overrides": []
+          },
+          "gridPos": { "h": 6, "w": 12, "x": 12, "y": 54 }
+        },
+        {
+          "id": 18,
+          "type": "timeseries",
+          "title": "RPC Slot & Reference Slot",
+          "datasource": { "type": "prometheus", "uid": "prometheus" },
+          "targets": [
+            { "refId": "T", "expr": "solana_rpc_slot{job=\"mpabi-node-exporter\",instance=\"$instance\"}" },
+            { "refId": "U", "expr": "solana_rpc_slot_reference{job=\"mpabi-node-exporter\",instance=\"$instance\"}" },
+            { "refId": "V", "expr": "solana_rpc_block_height{job=\"mpabi-node-exporter\",instance=\"$instance\"}" }
+          ],
+          "gridPos": { "h": 8, "w": 24, "x": 0, "y": 60 }
        }
      ]
    }
-
--- a/kustomize/infra/monitoring-extras/kustomization.yaml
+++ b/kustomize/infra/monitoring-extras/kustomization.yaml
@@ -8,3 +8,4 @@ resources:
  - ingressroute-prometheus.yaml
  - ingressroute-prometheus-http.yaml
  - dashboard-agave-status.yaml
+  - prometheus-rules-agave.yaml
--- a/kustomize/infra/monitoring-extras/prometheus-rules-agave.yaml
+++ b/kustomize/infra/monitoring-extras/prometheus-rules-agave.yaml
@@ -0,0 +1,61 @@
+apiVersion: monitoring.coreos.com/v1
+kind: PrometheusRule
+metadata:
+  name: agave-rpc-alerts
+  namespace: monitoring
+  labels:
+    app: kube-prometheus-stack
+    release: monitoring-stack
+spec:
+  groups:
+    - name: agave-rpc-health
+      rules:
+        - alert: AgaveRPCDown
+          expr: "max by (instance) (solana_rpc_up{job=\"mpabi-node-exporter\"}) == 0"
+          for: 30s
+          labels:
+            severity: critical
+            team: mpabi
+          annotations:
+            summary: "Agave RPC is unreachable"
+            description: "RPC probe from node exporter reports solana_rpc_up == 0 for instance {{ $labels.instance }}."
+        - alert: AgaveRPCSlotLagHigh
+          expr: "sum by (instance) (solana_rpc_slot_lag{job=\"mpabi-node-exporter\"}) > 50"
+          for: 2m
+          labels:
+            severity: warning
+            team: mpabi
+          annotations:
+            summary: "Agave RPC is lagging behind cluster"
+            description: "Current slot lag is {{ $value }} for instance {{ $labels.instance }}. Reference endpoint in probe config may be misconfigured or validator is behind."
+        - alert: AgaveRPCSlotLagCritical
+          expr: "sum by (instance) (solana_rpc_slot_lag{job=\"mpabi-node-exporter\"}) > 500"
+          for: 2m
+          labels:
+            severity: critical
+            team: mpabi
+          annotations:
+            summary: "Agave RPC severe lag"
+            description: "Slot lag is critically high ({{ $value }} slots) on instance {{ $labels.instance }}."
+        - alert: AgaveIOHigh
+          expr: |
+            sum by (instance) (
+              (rate(node_disk_read_bytes_total{job="mpabi-node-exporter",device=~"nvme.*"}[5m]) +
+               rate(node_disk_written_bytes_total{job="mpabi-node-exporter",device=~"nvme.*"}[5m])) / 1024 / 1024
+            ) > 300
+          for: 5m
+          labels:
+            severity: warning
+            team: mpabi
+          annotations:
+            summary: "High storage I/O on Agave node"
+            description: "Combined NVMe read+write throughput >300 MiB/s for 5m on {{ $labels.instance }}. Check disk pressure and Geyser/ledger workload."
+        - alert: AgaveIOWaitHigh
+          expr: "avg by (instance) (rate(node_cpu_seconds_total{job=\"mpabi-node-exporter\",mode=\"iowait\"}[5m])) > 0.2"
+          for: 5m
+          labels:
+            severity: warning
+            team: mpabi
+          annotations:
+            summary: "High iowait on Agave node"
+            description: "Iowait over 20% on average for 5m on {{ $labels.instance }}. Storage latency is likely impacting slot progress."
--- a/kustomize/overlays/prod/dlob-rpc-endpoint-patch.yaml
+++ b/kustomize/overlays/prod/dlob-rpc-endpoint-patch.yaml
@@ -0,0 +1,14 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dlob-publisher
+spec:
+  template:
+    spec:
+      containers:
+        - name: publisher
+          env:
+            - name: ENDPOINT
+              value: "http://10.66.66.1:8899"
+            - name: WS_ENDPOINT
+              value: "ws://10.66.66.1:8900"
--- a/kustomize/overlays/prod/dlob-rpc-server-endpoint-patch.yaml
+++ b/kustomize/overlays/prod/dlob-rpc-server-endpoint-patch.yaml
@@ -0,0 +1,14 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dlob-server
+spec:
+  template:
+    spec:
+      containers:
+        - name: server
+          env:
+            - name: ENDPOINT
+              value: "http://10.66.66.1:8899"
+            - name: WS_ENDPOINT
+              value: "ws://10.66.66.1:8900"
--- a/kustomize/overlays/prod/kustomization.yaml
+++ b/kustomize/overlays/prod/kustomization.yaml
@@ -7,6 +7,8 @@ resources:
  - ../../base

 patchesStrategicMerge:
+  - dlob-rpc-endpoint-patch.yaml
+  - dlob-rpc-server-endpoint-patch.yaml
  - frontend-graphql-proxy-patch.yaml

 configMapGenerator:
--- a/kustomize/overlays/staging/dlob-ingestor.mjs
+++ b/kustomize/overlays/staging/dlob-ingestor.mjs
@@ -66,7 +66,7 @@ function resolveConfig() {
  const hasuraAdminSecret = envString('HASURA_ADMIN_SECRET', '');
  if (!hasuraAdminSecret) throw new Error('Missing HASURA_ADMIN_SECRET');

-  const markets = envList('DLOB_MARKETS', 'PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP');
+  const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
  const pollMs = envInt('TICKS_POLL_MS', 1000, { min: 250, max: 60_000 });
  const source = envString('TICKS_SOURCE', 'dlob_stats');

--- a/kustomize/overlays/staging/dlob-rpc-endpoint-patch.yaml
+++ b/kustomize/overlays/staging/dlob-rpc-endpoint-patch.yaml
@@ -0,0 +1,14 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dlob-publisher
+spec:
+  template:
+    spec:
+      containers:
+        - name: publisher
+          env:
+            - name: ENDPOINT
+              value: "http://10.66.66.1:8899"
+            - name: WS_ENDPOINT
+              value: "ws://10.66.66.1:8900"
--- a/kustomize/overlays/staging/dlob-rpc-server-endpoint-patch.yaml
+++ b/kustomize/overlays/staging/dlob-rpc-server-endpoint-patch.yaml
@@ -0,0 +1,14 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dlob-server
+spec:
+  template:
+    spec:
+      containers:
+        - name: server
+          env:
+            - name: ENDPOINT
+              value: "http://10.66.66.1:8899"
+            - name: WS_ENDPOINT
+              value: "ws://10.66.66.1:8900"
--- a/kustomize/overlays/staging/ingestor-dlob-patch.yaml
+++ b/kustomize/overlays/staging/ingestor-dlob-patch.yaml
@@ -18,7 +18,7 @@ spec:
                  name: trade-hasura
                  key: HASURA_GRAPHQL_ADMIN_SECRET
            - name: DLOB_MARKETS
-              value: PUMP-PERP,SOL-PERP,1MBONK-PERP,BTC-PERP,ETH-PERP
+              value: SOL-PERP,DOGE-PERP,JUP-PERP
            - name: TICKS_POLL_MS
              value: "1000"
            - name: TICKS_SOURCE
--- a/kustomize/overlays/staging/kustomization.yaml
+++ b/kustomize/overlays/staging/kustomization.yaml
@@ -10,6 +10,8 @@ resources:
  - frontend-ingress-root.yaml

 patchesStrategicMerge:
+  - dlob-rpc-endpoint-patch.yaml
+  - dlob-rpc-server-endpoint-patch.yaml
  - hasura-patch.yaml
  - frontend-auth-patch.yaml
  - frontend-graphql-proxy-patch.yaml