131 lines
4.8 KiB
Bash
Executable File
131 lines
4.8 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
TARGET_HOST="${TARGET_HOST:-mevnode}"
|
|
TARGET_NAMESPACE="${TARGET_NAMESPACE:-trade-r001-canary}"
|
|
MAX_AGAVE_LAG="${MAX_AGAVE_LAG:-50}"
|
|
|
|
ssh_target() {
|
|
ssh -o StrictHostKeyChecking=no "$TARGET_HOST" "$@"
|
|
}
|
|
|
|
remote_bash() {
|
|
local script="$1"
|
|
ssh_target \
|
|
"TARGET_NAMESPACE=$(printf '%q' "$TARGET_NAMESPACE") MAX_AGAVE_LAG=$(printf '%q' "$MAX_AGAVE_LAG") bash -lc $(printf '%q' "$script")"
|
|
}
|
|
|
|
remote_bash_stdin() {
|
|
ssh_target \
|
|
"TARGET_NAMESPACE=$(printf '%q' "$TARGET_NAMESPACE") MAX_AGAVE_LAG=$(printf '%q' "$MAX_AGAVE_LAG") bash -s"
|
|
}
|
|
|
|
echo "[1/5] Host services"
|
|
remote_bash '
|
|
set -euo pipefail
|
|
echo "time=$(date --iso-8601=seconds)"
|
|
for svc in agave-validator k3s; do
|
|
state="$(systemctl is-active "$svc")"
|
|
echo "$svc=$state"
|
|
test "$state" = active
|
|
done
|
|
'
|
|
|
|
echo
|
|
echo "[2/5] Agave RPC lag"
|
|
remote_bash '
|
|
set -euo pipefail
|
|
req='\''{"jsonrpc":"2.0","id":1,"method":"getSlot"}'\''
|
|
health_req='\''{"jsonrpc":"2.0","id":1,"method":"getHealth"}'\''
|
|
local_slot="$(curl -fsS -H "Content-Type: application/json" --data-binary "$req" http://127.0.0.1:8899 | jq -r .result)"
|
|
ref_slot="$(curl -fsS -H "Content-Type: application/json" --data-binary "$req" https://api.mainnet-beta.solana.com | jq -r .result)"
|
|
health="$(curl -fsS -H "Content-Type: application/json" --data-binary "$health_req" http://127.0.0.1:8899 | jq -r ".result // .error.message")"
|
|
lag="$((ref_slot-local_slot))"
|
|
echo "local_slot=$local_slot"
|
|
echo "reference_slot=$ref_slot"
|
|
echo "lag=$lag"
|
|
echo "health=$health"
|
|
test "$health" = ok
|
|
test "$lag" -le "$MAX_AGAVE_LAG"
|
|
'
|
|
|
|
echo
|
|
echo "[3/5] Canary deployments"
|
|
remote_bash '
|
|
set -euo pipefail
|
|
sudo k3s kubectl -n "$TARGET_NAMESPACE" wait --for=condition=Available --timeout=180s deploy --all >/dev/null
|
|
sudo k3s kubectl -n "$TARGET_NAMESPACE" get deploy -o wide
|
|
bad="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get deploy -o json | jq -r ".items[] | select((.status.readyReplicas // 0) != (.spec.replicas // 1) or (.status.availableReplicas // 0) != (.spec.replicas // 1)) | .metadata.name")"
|
|
if [ -n "$bad" ]; then
|
|
echo "deployments_not_ready=$bad" >&2
|
|
exit 1
|
|
fi
|
|
'
|
|
|
|
echo
|
|
echo "[4/5] Database freshness"
|
|
remote_bash_stdin <<'REMOTE'
|
|
set -euo pipefail
|
|
pg_user="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get secret trade-postgres -o jsonpath="{.data.POSTGRES_USER}" | base64 -d)"
|
|
pg_pass="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get secret trade-postgres -o jsonpath="{.data.POSTGRES_PASSWORD}" | base64 -d)"
|
|
pg_db="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get secret trade-postgres -o jsonpath="{.data.POSTGRES_DB}" | base64 -d)"
|
|
export PGPASSWORD="$pg_pass"
|
|
sql="$(cat <<'SQL'
|
|
SELECT 'dlob_hot_derived_latest|' || count(*) FROM dlob_hot_derived_latest;
|
|
SELECT 'dlob_all_derived_latest|' || count(*) FROM dlob_all_derived_latest;
|
|
SELECT 'drift_ticks_15m|' || count(*) FROM drift_ticks WHERE ts >= now() - interval '15 minutes';
|
|
SELECT 'latest_tick|' || symbol || '|' || source || '|' || COALESCE(raw->>'from','<null>') || '|' || to_char(ts, 'YYYY-MM-DD HH24:MI:SS')
|
|
FROM drift_ticks
|
|
ORDER BY ts DESC
|
|
LIMIT 1;
|
|
SQL
|
|
)"
|
|
psql -h 127.0.0.1 -U "$pg_user" -d "$pg_db" -Atqc "$sql"
|
|
REMOTE
|
|
|
|
echo
|
|
echo "[5/5] API and frontend"
|
|
remote_bash_stdin <<'REMOTE'
|
|
set -euo pipefail
|
|
token="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get secret trade-frontend-tokens -o jsonpath="{.data.read\.json}" | base64 -d | jq -r .token)"
|
|
pod_name="$(sudo k3s kubectl -n "$TARGET_NAMESPACE" get pod -l app.kubernetes.io/name=trade-ingestor -o jsonpath="{.items[0].metadata.name}")"
|
|
sudo k3s kubectl -n "$TARGET_NAMESPACE" exec -i "$pod_name" -- env API_TOKEN="$token" node - <<'JS'
|
|
const headers = { Authorization: `Bearer ${process.env.API_TOKEN}` };
|
|
|
|
async function getJson(url) {
|
|
const response = await fetch(url, {
|
|
headers,
|
|
signal: AbortSignal.timeout(10000),
|
|
});
|
|
const payload = await response.json();
|
|
if (!response.ok || !payload?.ok) {
|
|
throw new Error(`${url} failed: ${response.status} ${JSON.stringify(payload)}`);
|
|
}
|
|
return payload;
|
|
}
|
|
|
|
(async () => {
|
|
const ticks = await getJson('http://trade-api:8787/v1/ticks?symbol=SOL-PERP&limit=3');
|
|
const chart = await getJson('http://trade-api:8787/v1/chart?symbol=SOL-PERP&tf=1m&limit=5');
|
|
const frontend = await fetch('http://trade-frontend:8081/', {
|
|
signal: AbortSignal.timeout(10000),
|
|
});
|
|
const html = await frontend.text();
|
|
if (!frontend.ok || !/<html/i.test(html)) {
|
|
throw new Error(`frontend failed: ${frontend.status}`);
|
|
}
|
|
console.log(JSON.stringify({
|
|
ticks_last: ticks.ticks?.at(-1) || null,
|
|
chart_last: chart.candles?.at(-1) || null,
|
|
frontend: { status: frontend.status, bytes: html.length },
|
|
}, null, 2));
|
|
})().catch((err) => {
|
|
console.error(String(err && err.message ? err.message : err));
|
|
process.exit(1);
|
|
});
|
|
JS
|
|
REMOTE
|
|
|
|
echo
|
|
echo "Smoke check passed for ${TARGET_NAMESPACE} on ${TARGET_HOST}"
|