220 lines
8.4 KiB
YAML
220 lines
8.4 KiB
YAML
name: deploy-trade-r001-canary
|
|
|
|
on:
|
|
push:
|
|
branches:
|
|
- main
|
|
paths:
|
|
- environments/sol/trade-r001-canary/**
|
|
- .gitea/workflows/deploy-trade-r001-canary.yaml
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
apply:
|
|
runs-on: k3s-deploy
|
|
steps:
|
|
- name: Checkout repository
|
|
uses: actions/checkout@v4
|
|
|
|
- name: Materialize kubeconfig
|
|
env:
|
|
K3S_KUBECONFIG_B64: ${{ secrets.K3S_KUBECONFIG_B64 }}
|
|
run: |
|
|
test -n "$K3S_KUBECONFIG_B64"
|
|
printf '%s' "$K3S_KUBECONFIG_B64" | base64 -d >/tmp/kubeconfig
|
|
chmod 600 /tmp/kubeconfig
|
|
|
|
- name: Install kubectl
|
|
run: |
|
|
curl -fsSL -o /tmp/kubectl https://dl.k8s.io/release/v1.34.6/bin/linux/amd64/kubectl
|
|
install -m 0755 /tmp/kubectl /usr/local/bin/kubectl
|
|
kubectl version --client
|
|
|
|
- name: Verify prerequisite secrets
|
|
env:
|
|
KUBECONFIG: /tmp/kubeconfig
|
|
run: |
|
|
kubectl -n trade-r001-canary get secret trade-postgres trade-hasura trade-api trade-frontend-tokens trade-basic-auth trade-ingestor-tokens trade-dlob-rpc gitea-registry
|
|
|
|
- name: Recreate bootstrap jobs
|
|
env:
|
|
KUBECONFIG: /tmp/kubeconfig
|
|
run: |
|
|
kubectl -n trade-r001-canary delete job postgres-migrate hasura-bootstrap --ignore-not-found=true
|
|
|
|
- name: Apply shared host access infrastructure
|
|
env:
|
|
KUBECONFIG: /tmp/kubeconfig
|
|
run: |
|
|
kubectl apply -k environments/sol/trade-infra
|
|
kubectl -n trade-infra get svc,endpointslice
|
|
|
|
- name: Apply canary environment
|
|
env:
|
|
KUBECONFIG: /tmp/kubeconfig
|
|
run: |
|
|
kubectl apply -k environments/sol/trade-r001-canary
|
|
kubectl get ns trade-r001-canary --show-labels
|
|
kubectl -n trade-r001-canary get svc,resourcequota,limitrange
|
|
|
|
- name: Restart application surface
|
|
env:
|
|
KUBECONFIG: /tmp/kubeconfig
|
|
run: |
|
|
kubectl -n trade-r001-canary rollout restart \
|
|
deploy/hasura \
|
|
deploy/trade-api \
|
|
deploy/trade-frontend \
|
|
deploy/trade-ingestor \
|
|
deploy/dlob-publisher-hot \
|
|
deploy/dlob-hot-redis-to-postgres-raw-writer \
|
|
deploy/dlob-hot-postgres-to-postgres-derived-writer
|
|
|
|
- name: Wait for database and metadata bootstrap
|
|
env:
|
|
KUBECONFIG: /tmp/kubeconfig
|
|
run: |
|
|
kubectl -n trade-r001-canary wait --for=condition=complete job/postgres-migrate --timeout=300s
|
|
kubectl -n trade-r001-canary wait --for=condition=complete job/hasura-bootstrap --timeout=300s
|
|
|
|
- name: Wait for application rollouts
|
|
env:
|
|
KUBECONFIG: /tmp/kubeconfig
|
|
run: |
|
|
kubectl -n trade-r001-canary rollout status deploy/hasura --timeout=300s
|
|
kubectl -n trade-r001-canary rollout status deploy/trade-api --timeout=300s
|
|
kubectl -n trade-r001-canary rollout status deploy/trade-frontend --timeout=300s
|
|
kubectl -n trade-r001-canary rollout status deploy/trade-ingestor --timeout=300s
|
|
kubectl -n trade-r001-canary rollout status deploy/dlob-publisher-hot --timeout=420s
|
|
kubectl -n trade-r001-canary rollout status deploy/dlob-hot-redis-to-postgres-raw-writer --timeout=300s
|
|
kubectl -n trade-r001-canary rollout status deploy/dlob-hot-postgres-to-postgres-derived-writer --timeout=300s
|
|
kubectl -n trade-r001-canary get deploy,pods -o wide
|
|
|
|
- name: Verify trade-ingestor runtime
|
|
env:
|
|
KUBECONFIG: /tmp/kubeconfig
|
|
run: |
|
|
sleep 10
|
|
pod_name="$(kubectl -n trade-r001-canary get pod -l app.kubernetes.io/name=trade-ingestor -o jsonpath='{.items[0].metadata.name}')"
|
|
restart_count="$(kubectl -n trade-r001-canary get pod "$pod_name" -o jsonpath='{.status.containerStatuses[0].restartCount}')"
|
|
test "${restart_count}" = "0"
|
|
kubectl -n trade-r001-canary logs "$pod_name" --tail=20
|
|
|
|
- name: Verify canary namespace connectivity
|
|
env:
|
|
KUBECONFIG: /tmp/kubeconfig
|
|
run: |
|
|
pod_name="$(kubectl -n trade-r001-canary get pod -l app.kubernetes.io/name=trade-ingestor -o jsonpath='{.items[0].metadata.name}')"
|
|
kubectl -n trade-r001-canary exec -i "$pod_name" -- node - <<'JS'
|
|
const net = require('net');
|
|
|
|
const targets = [
|
|
['postgres-host.trade-infra.svc.cluster.local', 5432],
|
|
['redis-host.trade-infra.svc.cluster.local', 6379],
|
|
['agave-rpc-host.trade-infra.svc.cluster.local', 8899],
|
|
['agave-ws-host.trade-infra.svc.cluster.local', 8900],
|
|
['agave-grpc-host.trade-infra.svc.cluster.local', 10000],
|
|
];
|
|
|
|
function checkSocket(host, port) {
|
|
return new Promise((resolve, reject) => {
|
|
const socket = net.createConnection({ host, port, timeout: 5000 });
|
|
socket.on('connect', () => {
|
|
console.log(`OK ${host}:${port}`);
|
|
socket.end();
|
|
resolve();
|
|
});
|
|
socket.on('timeout', () => {
|
|
socket.destroy(new Error(`Timeout ${host}:${port}`));
|
|
});
|
|
socket.on('error', reject);
|
|
});
|
|
}
|
|
|
|
(async () => {
|
|
for (const [host, port] of targets) {
|
|
await checkSocket(host, port);
|
|
}
|
|
})().catch((err) => {
|
|
console.error(String(err && err.message ? err.message : err));
|
|
process.exit(1);
|
|
});
|
|
JS
|
|
|
|
- name: Verify DLOB hot-path runtime
|
|
env:
|
|
KUBECONFIG: /tmp/kubeconfig
|
|
run: |
|
|
kubectl -n trade-r001-canary logs deploy/dlob-publisher-hot --tail=20
|
|
kubectl -n trade-r001-canary logs deploy/dlob-hot-redis-to-postgres-raw-writer --tail=20
|
|
kubectl -n trade-r001-canary logs deploy/dlob-hot-postgres-to-postgres-derived-writer --tail=20
|
|
pod_name="$(kubectl -n trade-r001-canary get pod -l app.kubernetes.io/name=trade-ingestor -o jsonpath='{.items[0].metadata.name}')"
|
|
kubectl -n trade-r001-canary exec -i "$pod_name" -- node - <<'JS'
|
|
const endpoint = 'http://hasura:8080/v1/graphql';
|
|
const adminSecret = process.env.HASURA_ADMIN_SECRET;
|
|
const query = `
|
|
query {
|
|
dlob_hot_derived_latest(limit: 1, order_by: { updated_at: desc }) {
|
|
market_name
|
|
updated_at
|
|
source
|
|
}
|
|
}
|
|
`;
|
|
|
|
async function check() {
|
|
const response = await fetch(endpoint, {
|
|
method: 'POST',
|
|
headers: {
|
|
'content-type': 'application/json',
|
|
'x-hasura-admin-secret': adminSecret,
|
|
},
|
|
body: JSON.stringify({ query }),
|
|
signal: AbortSignal.timeout(10000),
|
|
});
|
|
const payload = await response.json();
|
|
const rows = payload?.data?.dlob_hot_derived_latest || [];
|
|
if (!rows.length) {
|
|
throw new Error('No rows in dlob_hot_derived_latest yet');
|
|
}
|
|
console.log(JSON.stringify(rows[0], null, 2));
|
|
}
|
|
|
|
(async () => {
|
|
for (let attempt = 0; attempt < 24; attempt += 1) {
|
|
try {
|
|
await check();
|
|
return;
|
|
} catch (error) {
|
|
if (attempt === 23) {
|
|
throw error;
|
|
}
|
|
await new Promise((resolve) => setTimeout(resolve, 5000));
|
|
}
|
|
}
|
|
})().catch((error) => {
|
|
console.error(String(error && error.message ? error.message : error));
|
|
process.exit(1);
|
|
});
|
|
JS
|
|
kubectl -n trade-r001-canary exec -i "$pod_name" -- node - <<'JS'
|
|
const targets = [
|
|
'http://hasura:8080/healthz',
|
|
'http://trade-api:8787/healthz',
|
|
'http://trade-frontend:8081/healthz',
|
|
];
|
|
|
|
(async () => {
|
|
for (const url of targets) {
|
|
const response = await fetch(url, { signal: AbortSignal.timeout(10000) });
|
|
if (!response.ok) {
|
|
throw new Error(`Unexpected status for ${url}: ${response.status}`);
|
|
}
|
|
console.log(`OK ${url}`);
|
|
}
|
|
})().catch((err) => {
|
|
console.error(String(err && err.message ? err.message : err));
|
|
process.exit(1);
|
|
});
|
|
JS
|