#!/usr/bin/env bash set -euo pipefail TARGET_HOST="${TARGET_HOST:-mevnode}" TARGET_NAMESPACE="${TARGET_NAMESPACE:-trade-r001-canary}" MAX_AGAVE_LAG="${MAX_AGAVE_LAG:-50}" ssh_target() { ssh -o StrictHostKeyChecking=no "$TARGET_HOST" "$@" } remote_bash() { local script="$1" ssh_target \ "TARGET_NAMESPACE=$(printf '%q' "$TARGET_NAMESPACE") MAX_AGAVE_LAG=$(printf '%q' "$MAX_AGAVE_LAG") bash -lc $(printf '%q' "$script")" } remote_bash_stdin() { ssh_target \ "TARGET_NAMESPACE=$(printf '%q' "$TARGET_NAMESPACE") MAX_AGAVE_LAG=$(printf '%q' "$MAX_AGAVE_LAG") bash -s" } echo "[1/5] Host services" remote_bash ' set -euo pipefail echo "time=$(date --iso-8601=seconds)" for svc in agave-validator k3s; do state="$(systemctl is-active "$svc")" echo "$svc=$state" test "$state" = active done ' echo echo "[2/5] Agave RPC lag" remote_bash ' set -euo pipefail req='\''{"jsonrpc":"2.0","id":1,"method":"getSlot"}'\'' health_req='\''{"jsonrpc":"2.0","id":1,"method":"getHealth"}'\'' local_slot="$(curl -fsS -H "Content-Type: application/json" --data-binary "$req" http://127.0.0.1:8899 | jq -r .result)" ref_slot="$(curl -fsS -H "Content-Type: application/json" --data-binary "$req" https://api.mainnet-beta.solana.com | jq -r .result)" health="$(curl -fsS -H "Content-Type: application/json" --data-binary "$health_req" http://127.0.0.1:8899 | jq -r ".result // .error.message")" lag="$((ref_slot-local_slot))" echo "local_slot=$local_slot" echo "reference_slot=$ref_slot" echo "lag=$lag" echo "health=$health" test "$health" = ok test "$lag" -le "$MAX_AGAVE_LAG" ' echo echo "[3/5] Canary deployments" remote_bash ' set -euo pipefail sudo k3s kubectl -n "$TARGET_NAMESPACE" wait --for=condition=Available --timeout=180s deploy --all >/dev/null sudo k3s kubectl -n "$TARGET_NAMESPACE" get deploy -o wide bad="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get deploy -o json | jq -r ".items[] | select((.status.readyReplicas // 0) != (.spec.replicas // 1) or (.status.availableReplicas // 0) != (.spec.replicas // 1)) | .metadata.name")" if [ -n "$bad" ]; then echo "deployments_not_ready=$bad" >&2 exit 1 fi ' echo echo "[4/5] Database freshness" remote_bash_stdin <<'REMOTE' set -euo pipefail pg_user="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get secret trade-postgres -o jsonpath="{.data.POSTGRES_USER}" | base64 -d)" pg_pass="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get secret trade-postgres -o jsonpath="{.data.POSTGRES_PASSWORD}" | base64 -d)" pg_db="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get secret trade-postgres -o jsonpath="{.data.POSTGRES_DB}" | base64 -d)" export PGPASSWORD="$pg_pass" sql="$(cat <<'SQL' SELECT 'dlob_hot_derived_latest|' || count(*) FROM dlob_hot_derived_latest; SELECT 'dlob_all_derived_latest|' || count(*) FROM dlob_all_derived_latest; SELECT 'drift_ticks_15m|' || count(*) FROM drift_ticks WHERE ts >= now() - interval '15 minutes'; SELECT 'latest_tick|' || symbol || '|' || source || '|' || COALESCE(raw->>'from','') || '|' || to_char(ts, 'YYYY-MM-DD HH24:MI:SS') FROM drift_ticks ORDER BY ts DESC LIMIT 1; SQL )" psql -h 127.0.0.1 -U "$pg_user" -d "$pg_db" -Atqc "$sql" REMOTE echo echo "[5/5] API and frontend" remote_bash_stdin <<'REMOTE' set -euo pipefail token="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get secret trade-frontend-tokens -o jsonpath="{.data.read\.json}" | base64 -d | jq -r .token)" pod_name="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get pod -l app.kubernetes.io/name=trade-ingestor -o jsonpath="{.items[0].metadata.name}")" sudo k3s kubectl -n "$TARGET_NAMESPACE" exec -i "$pod_name" -- env API_TOKEN="$token" node - <<'JS' const headers = { Authorization: `Bearer ${process.env.API_TOKEN}` }; async function getJson(url) { const response = await fetch(url, { headers, signal: AbortSignal.timeout(10000), }); const payload = await response.json(); if (!response.ok || !payload?.ok) { throw new Error(`${url} failed: ${response.status} ${JSON.stringify(payload)}`); } return payload; } (async () => { const ticks = await getJson('http://trade-api:8787/v1/ticks?symbol=SOL-PERP&limit=3'); const chart = await getJson('http://trade-api:8787/v1/chart?symbol=SOL-PERP&tf=1m&limit=5'); const frontend = await fetch('http://trade-frontend:8081/', { signal: AbortSignal.timeout(10000), }); const html = await frontend.text(); if (!frontend.ok || !/ { console.error(String(err && err.message ? err.message : err)); process.exit(1); }); JS REMOTE echo echo "Smoke check passed for ${TARGET_NAMESPACE} on ${TARGET_HOST}"