diff --git a/.gitea/workflows/deploy-trade-r001-canary.yaml b/.gitea/workflows/deploy-trade-r001-canary.yaml index 897c5e4..35aaaad 100644 --- a/.gitea/workflows/deploy-trade-r001-canary.yaml +++ b/.gitea/workflows/deploy-trade-r001-canary.yaml @@ -58,20 +58,26 @@ jobs: kubectl get ns trade-r001-canary --show-labels kubectl -n trade-r001-canary get svc,resourcequota,limitrange - - name: Restart application surface + - name: Restart application surface sequentially env: KUBECONFIG: /tmp/kubeconfig run: | - kubectl -n trade-r001-canary rollout restart \ - deploy/hasura \ - deploy/trade-api \ - deploy/trade-frontend \ - deploy/trade-ingestor \ - deploy/dlob-publisher-hot \ - deploy/dlob-publisher-all \ - deploy/dlob-hot-redis-to-postgres-raw-writer \ - deploy/dlob-hot-postgres-to-postgres-derived-writer \ - deploy/dlob-all-redis-to-postgres-derived-writer + restart_and_wait() { + name="$1" + timeout="$2" + kubectl -n trade-r001-canary rollout restart "deploy/${name}" + kubectl -n trade-r001-canary rollout status "deploy/${name}" --timeout="${timeout}" + } + + restart_and_wait hasura 300s + restart_and_wait trade-api 300s + restart_and_wait trade-frontend 300s + restart_and_wait trade-ingestor 300s + restart_and_wait dlob-hot-redis-to-postgres-raw-writer 300s + restart_and_wait dlob-hot-postgres-to-postgres-derived-writer 300s + restart_and_wait dlob-all-redis-to-postgres-derived-writer 300s + restart_and_wait dlob-publisher-hot 420s + restart_and_wait dlob-publisher-all 420s - name: Wait for database and metadata bootstrap env: @@ -85,15 +91,6 @@ jobs: env: KUBECONFIG: /tmp/kubeconfig run: | - kubectl -n trade-r001-canary rollout status deploy/hasura --timeout=300s - kubectl -n trade-r001-canary rollout status deploy/trade-api --timeout=300s - kubectl -n trade-r001-canary rollout status deploy/trade-frontend --timeout=300s - kubectl -n trade-r001-canary rollout status deploy/trade-ingestor --timeout=300s - kubectl -n trade-r001-canary rollout status deploy/dlob-publisher-hot --timeout=420s - kubectl -n trade-r001-canary rollout status deploy/dlob-publisher-all --timeout=420s - kubectl -n trade-r001-canary rollout status deploy/dlob-hot-redis-to-postgres-raw-writer --timeout=300s - kubectl -n trade-r001-canary rollout status deploy/dlob-hot-postgres-to-postgres-derived-writer --timeout=300s - kubectl -n trade-r001-canary rollout status deploy/dlob-all-redis-to-postgres-derived-writer --timeout=300s kubectl -n trade-r001-canary get deploy,pods -o wide - name: Verify trade-ingestor runtime diff --git a/environments/sol/trade-r001-canary/README.md b/environments/sol/trade-r001-canary/README.md index 224c0c3..d6036fb 100644 --- a/environments/sol/trade-r001-canary/README.md +++ b/environments/sol/trade-r001-canary/README.md @@ -54,6 +54,7 @@ Minimal canary namespace for migration baseline `R001` on `sol`. before it waits for `Hasura`, `trade-api`, `trade-frontend`, `trade-ingestor`, and the DLOB hot/all-path deployments to become healthy. - The canary `trade-ingestor` now follows the live `R001` path on `sol`: it reads `dlob_hot_derived_latest` first for hot markets and falls back to `dlob_all_derived_latest`. - `api-token-seed` restores the frontend read token in `api_tokens`, so `trade-api` and `trade-frontend` can be validated against the reconstructed derived tables after each deploy. +- Canary deployments use `RollingUpdate` with `maxSurge=0` and `maxUnavailable=1`, and the workflow restarts them sequentially, because the namespace `limits.cpu=6` guardrail does not tolerate surge-based restarts of the full surface at once. ## Operator Flow diff --git a/environments/sol/trade-r001-canary/api-token-seed-job.yaml b/environments/sol/trade-r001-canary/api-token-seed-job.yaml index 7a3eb21..63e4f6e 100644 --- a/environments/sol/trade-r001-canary/api-token-seed-job.yaml +++ b/environments/sol/trade-r001-canary/api-token-seed-job.yaml @@ -12,6 +12,13 @@ spec: - name: seed image: postgres:16-alpine imagePullPolicy: IfNotPresent + resources: + requests: + cpu: 100m + memory: 64Mi + limits: + cpu: 250m + memory: 128Mi command: - sh - -lc diff --git a/environments/sol/trade-r001-canary/dlob-hot-postgres-to-postgres-derived-writer-deployment.yaml b/environments/sol/trade-r001-canary/dlob-hot-postgres-to-postgres-derived-writer-deployment.yaml index 1a88d80..95f2629 100644 --- a/environments/sol/trade-r001-canary/dlob-hot-postgres-to-postgres-derived-writer-deployment.yaml +++ b/environments/sol/trade-r001-canary/dlob-hot-postgres-to-postgres-derived-writer-deployment.yaml @@ -5,6 +5,11 @@ metadata: namespace: trade-r001-canary spec: replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: dlob-hot-postgres-to-postgres-derived-writer diff --git a/environments/sol/trade-r001-canary/dlob-hot-redis-to-postgres-raw-writer-deployment.yaml b/environments/sol/trade-r001-canary/dlob-hot-redis-to-postgres-raw-writer-deployment.yaml index bc63889..397d83a 100644 --- a/environments/sol/trade-r001-canary/dlob-hot-redis-to-postgres-raw-writer-deployment.yaml +++ b/environments/sol/trade-r001-canary/dlob-hot-redis-to-postgres-raw-writer-deployment.yaml @@ -5,6 +5,11 @@ metadata: namespace: trade-r001-canary spec: replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: dlob-hot-redis-to-postgres-raw-writer diff --git a/environments/sol/trade-r001-canary/dlob-publisher-hot-deployment.yaml b/environments/sol/trade-r001-canary/dlob-publisher-hot-deployment.yaml index b3cb487..eda7e9f 100644 --- a/environments/sol/trade-r001-canary/dlob-publisher-hot-deployment.yaml +++ b/environments/sol/trade-r001-canary/dlob-publisher-hot-deployment.yaml @@ -5,6 +5,11 @@ metadata: namespace: trade-r001-canary spec: replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: dlob-publisher-hot diff --git a/environments/sol/trade-r001-canary/hasura-deployment.yaml b/environments/sol/trade-r001-canary/hasura-deployment.yaml index 759ff7b..470a7f3 100644 --- a/environments/sol/trade-r001-canary/hasura-deployment.yaml +++ b/environments/sol/trade-r001-canary/hasura-deployment.yaml @@ -5,6 +5,11 @@ metadata: namespace: trade-r001-canary spec: replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app: hasura diff --git a/environments/sol/trade-r001-canary/trade-api-deployment.yaml b/environments/sol/trade-r001-canary/trade-api-deployment.yaml index fdf499c..0e81a42 100644 --- a/environments/sol/trade-r001-canary/trade-api-deployment.yaml +++ b/environments/sol/trade-r001-canary/trade-api-deployment.yaml @@ -5,6 +5,11 @@ metadata: namespace: trade-r001-canary spec: replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: trade-api diff --git a/environments/sol/trade-r001-canary/trade-frontend-deployment.yaml b/environments/sol/trade-r001-canary/trade-frontend-deployment.yaml index 0a8444f..1f1bac5 100644 --- a/environments/sol/trade-r001-canary/trade-frontend-deployment.yaml +++ b/environments/sol/trade-r001-canary/trade-frontend-deployment.yaml @@ -5,6 +5,11 @@ metadata: namespace: trade-r001-canary spec: replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: trade-frontend diff --git a/environments/sol/trade-r001-canary/trade-ingestor-deployment.yaml b/environments/sol/trade-r001-canary/trade-ingestor-deployment.yaml index 8db8c33..cd701e0 100644 --- a/environments/sol/trade-r001-canary/trade-ingestor-deployment.yaml +++ b/environments/sol/trade-r001-canary/trade-ingestor-deployment.yaml @@ -5,6 +5,11 @@ metadata: namespace: trade-r001-canary spec: replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxSurge: 0 + maxUnavailable: 1 selector: matchLabels: app.kubernetes.io/name: trade-ingestor