fix(sol): serialize canary rollouts under quota
All checks were successful
deploy-trade-r001-canary / apply (push) Successful in 6m46s

This commit is contained in:
mpabi
2026-04-12 18:39:31 +02:00
parent 2e909026a7
commit c76eb7d5f3
10 changed files with 60 additions and 20 deletions

View File

@@ -58,20 +58,26 @@ jobs:
kubectl get ns trade-r001-canary --show-labels
kubectl -n trade-r001-canary get svc,resourcequota,limitrange
- name: Restart application surface
- name: Restart application surface sequentially
env:
KUBECONFIG: /tmp/kubeconfig
run: |
kubectl -n trade-r001-canary rollout restart \
deploy/hasura \
deploy/trade-api \
deploy/trade-frontend \
deploy/trade-ingestor \
deploy/dlob-publisher-hot \
deploy/dlob-publisher-all \
deploy/dlob-hot-redis-to-postgres-raw-writer \
deploy/dlob-hot-postgres-to-postgres-derived-writer \
deploy/dlob-all-redis-to-postgres-derived-writer
restart_and_wait() {
name="$1"
timeout="$2"
kubectl -n trade-r001-canary rollout restart "deploy/${name}"
kubectl -n trade-r001-canary rollout status "deploy/${name}" --timeout="${timeout}"
}
restart_and_wait hasura 300s
restart_and_wait trade-api 300s
restart_and_wait trade-frontend 300s
restart_and_wait trade-ingestor 300s
restart_and_wait dlob-hot-redis-to-postgres-raw-writer 300s
restart_and_wait dlob-hot-postgres-to-postgres-derived-writer 300s
restart_and_wait dlob-all-redis-to-postgres-derived-writer 300s
restart_and_wait dlob-publisher-hot 420s
restart_and_wait dlob-publisher-all 420s
- name: Wait for database and metadata bootstrap
env:
@@ -85,15 +91,6 @@ jobs:
env:
KUBECONFIG: /tmp/kubeconfig
run: |
kubectl -n trade-r001-canary rollout status deploy/hasura --timeout=300s
kubectl -n trade-r001-canary rollout status deploy/trade-api --timeout=300s
kubectl -n trade-r001-canary rollout status deploy/trade-frontend --timeout=300s
kubectl -n trade-r001-canary rollout status deploy/trade-ingestor --timeout=300s
kubectl -n trade-r001-canary rollout status deploy/dlob-publisher-hot --timeout=420s
kubectl -n trade-r001-canary rollout status deploy/dlob-publisher-all --timeout=420s
kubectl -n trade-r001-canary rollout status deploy/dlob-hot-redis-to-postgres-raw-writer --timeout=300s
kubectl -n trade-r001-canary rollout status deploy/dlob-hot-postgres-to-postgres-derived-writer --timeout=300s
kubectl -n trade-r001-canary rollout status deploy/dlob-all-redis-to-postgres-derived-writer --timeout=300s
kubectl -n trade-r001-canary get deploy,pods -o wide
- name: Verify trade-ingestor runtime

View File

@@ -54,6 +54,7 @@ Minimal canary namespace for migration baseline `R001` on `sol`.
before it waits for `Hasura`, `trade-api`, `trade-frontend`, `trade-ingestor`, and the DLOB hot/all-path deployments to become healthy.
- The canary `trade-ingestor` now follows the live `R001` path on `sol`: it reads `dlob_hot_derived_latest` first for hot markets and falls back to `dlob_all_derived_latest`.
- `api-token-seed` restores the frontend read token in `api_tokens`, so `trade-api` and `trade-frontend` can be validated against the reconstructed derived tables after each deploy.
- Canary deployments use `RollingUpdate` with `maxSurge=0` and `maxUnavailable=1`, and the workflow restarts them sequentially, because the namespace `limits.cpu=6` guardrail does not tolerate surge-based restarts of the full surface at once.
## Operator Flow

View File

@@ -12,6 +12,13 @@ spec:
- name: seed
image: postgres:16-alpine
imagePullPolicy: IfNotPresent
resources:
requests:
cpu: 100m
memory: 64Mi
limits:
cpu: 250m
memory: 128Mi
command:
- sh
- -lc

View File

@@ -5,6 +5,11 @@ metadata:
namespace: trade-r001-canary
spec:
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-hot-postgres-to-postgres-derived-writer

View File

@@ -5,6 +5,11 @@ metadata:
namespace: trade-r001-canary
spec:
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-hot-redis-to-postgres-raw-writer

View File

@@ -5,6 +5,11 @@ metadata:
namespace: trade-r001-canary
spec:
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: dlob-publisher-hot

View File

@@ -5,6 +5,11 @@ metadata:
namespace: trade-r001-canary
spec:
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app: hasura

View File

@@ -5,6 +5,11 @@ metadata:
namespace: trade-r001-canary
spec:
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: trade-api

View File

@@ -5,6 +5,11 @@ metadata:
namespace: trade-r001-canary
spec:
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: trade-frontend

View File

@@ -5,6 +5,11 @@ metadata:
namespace: trade-r001-canary
spec:
replicas: 1
strategy:
type: RollingUpdate
rollingUpdate:
maxSurge: 0
maxUnavailable: 1
selector:
matchLabels:
app.kubernetes.io/name: trade-ingestor