ops(sol): add offline canary recovery path

environments/sol/trade-r001-canary/scripts/check-sol-canary-smoke.sh

@@ -0,0 +1,130 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+TARGET_HOST="${TARGET_HOST:-mevnode}"
+TARGET_NAMESPACE="${TARGET_NAMESPACE:-trade-r001-canary}"
+MAX_AGAVE_LAG="${MAX_AGAVE_LAG:-50}"
+
+ssh_target() {
+  ssh -o StrictHostKeyChecking=no "$TARGET_HOST" "$@"
+}
+
+remote_bash() {
+  local script="$1"
+  ssh_target \
+    "TARGET_NAMESPACE=$(printf '%q' "$TARGET_NAMESPACE") MAX_AGAVE_LAG=$(printf '%q' "$MAX_AGAVE_LAG") bash -lc $(printf '%q' "$script")"
+}
+
+remote_bash_stdin() {
+  ssh_target \
+    "TARGET_NAMESPACE=$(printf '%q' "$TARGET_NAMESPACE") MAX_AGAVE_LAG=$(printf '%q' "$MAX_AGAVE_LAG") bash -s"
+}
+
+echo "[1/5] Host services"
+remote_bash '
+  set -euo pipefail
+  echo "time=$(date --iso-8601=seconds)"
+  for svc in agave-validator k3s; do
+    state="$(systemctl is-active "$svc")"
+    echo "$svc=$state"
+    test "$state" = active
+  done
+'
+
+echo
+echo "[2/5] Agave RPC lag"
+remote_bash '
+  set -euo pipefail
+  req='\''{"jsonrpc":"2.0","id":1,"method":"getSlot"}'\''
+  health_req='\''{"jsonrpc":"2.0","id":1,"method":"getHealth"}'\''
+  local_slot="$(curl -fsS -H "Content-Type: application/json" --data-binary "$req" http://127.0.0.1:8899 | jq -r .result)"
+  ref_slot="$(curl -fsS -H "Content-Type: application/json" --data-binary "$req" https://api.mainnet-beta.solana.com | jq -r .result)"
+  health="$(curl -fsS -H "Content-Type: application/json" --data-binary "$health_req" http://127.0.0.1:8899 | jq -r ".result // .error.message")"
+  lag="$((ref_slot-local_slot))"
+  echo "local_slot=$local_slot"
+  echo "reference_slot=$ref_slot"
+  echo "lag=$lag"
+  echo "health=$health"
+  test "$health" = ok
+  test "$lag" -le "$MAX_AGAVE_LAG"
+'
+
+echo
+echo "[3/5] Canary deployments"
+remote_bash '
+  set -euo pipefail
+  sudo k3s kubectl -n "$TARGET_NAMESPACE" wait --for=condition=Available --timeout=180s deploy --all >/dev/null
+  sudo k3s kubectl -n "$TARGET_NAMESPACE" get deploy -o wide
+  bad="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get deploy -o json | jq -r ".items[] | select((.status.readyReplicas // 0) != (.spec.replicas // 1) or (.status.availableReplicas // 0) != (.spec.replicas // 1)) | .metadata.name")"
+  if [ -n "$bad" ]; then
+    echo "deployments_not_ready=$bad" >&2
+    exit 1
+  fi
+'
+
+echo
+echo "[4/5] Database freshness"
+remote_bash_stdin <<'REMOTE'
+  set -euo pipefail
+  pg_user="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get secret trade-postgres -o jsonpath="{.data.POSTGRES_USER}" | base64 -d)"
+  pg_pass="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get secret trade-postgres -o jsonpath="{.data.POSTGRES_PASSWORD}" | base64 -d)"
+  pg_db="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get secret trade-postgres -o jsonpath="{.data.POSTGRES_DB}" | base64 -d)"
+  export PGPASSWORD="$pg_pass"
+  sql="$(cat <<'SQL'
+SELECT 'dlob_hot_derived_latest|' || count(*) FROM dlob_hot_derived_latest;
+SELECT 'dlob_all_derived_latest|' || count(*) FROM dlob_all_derived_latest;
+SELECT 'drift_ticks_15m|' || count(*) FROM drift_ticks WHERE ts >= now() - interval '15 minutes';
+SELECT 'latest_tick|' || symbol || '|' || source || '|' || COALESCE(raw->>'from','<null>') || '|' || to_char(ts, 'YYYY-MM-DD HH24:MI:SS')
+FROM drift_ticks
+ORDER BY ts DESC
+LIMIT 1;
+SQL
+)"
+  psql -h 127.0.0.1 -U "$pg_user" -d "$pg_db" -Atqc "$sql"
+REMOTE
+
+echo
+echo "[5/5] API and frontend"
+remote_bash_stdin <<'REMOTE'
+  set -euo pipefail
+  token="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get secret trade-frontend-tokens -o jsonpath="{.data.read\.json}" | base64 -d | jq -r .token)"
+  pod_name="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get pod -l app.kubernetes.io/name=trade-ingestor -o jsonpath="{.items[0].metadata.name}")"
+  sudo k3s kubectl -n "$TARGET_NAMESPACE" exec -i "$pod_name" -- env API_TOKEN="$token" node - <<'JS'
+const headers = { Authorization: `Bearer ${process.env.API_TOKEN}` };
+
+async function getJson(url) {
+  const response = await fetch(url, {
+    headers,
+    signal: AbortSignal.timeout(10000),
+  });
+  const payload = await response.json();
+  if (!response.ok || !payload?.ok) {
+    throw new Error(`${url} failed: ${response.status} ${JSON.stringify(payload)}`);
+  }
+  return payload;
+}
+
+(async () => {
+  const ticks = await getJson('http://trade-api:8787/v1/ticks?symbol=SOL-PERP&limit=3');
+  const chart = await getJson('http://trade-api:8787/v1/chart?symbol=SOL-PERP&tf=1m&limit=5');
+  const frontend = await fetch('http://trade-frontend:8081/', {
+    signal: AbortSignal.timeout(10000),
+  });
+  const html = await frontend.text();
+  if (!frontend.ok || !/<html/i.test(html)) {
+    throw new Error(`frontend failed: ${frontend.status}`);
+  }
+  console.log(JSON.stringify({
+    ticks_last: ticks.ticks?.at(-1) || null,
+    chart_last: chart.candles?.at(-1) || null,
+    frontend: { status: frontend.status, bytes: html.length },
+  }, null, 2));
+})().catch((err) => {
+  console.error(String(err && err.message ? err.message : err));
+  process.exit(1);
+});
+JS
+REMOTE
+
+echo
+echo "Smoke check passed for ${TARGET_NAMESPACE} on ${TARGET_HOST}"