Compare commits
99 Commits
e7d4d405c3
...
snapshot/2
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4354de5c83 | ||
|
|
71b00f80bb | ||
|
|
01f6543ae9 | ||
|
|
8a474242df | ||
|
|
db4e45dc55 | ||
|
|
3572382c90 | ||
|
|
bb6f633d19 | ||
|
|
108bf22968 | ||
|
|
59c3f3ee06 | ||
|
|
9c4c3096d7 | ||
|
|
b02bd6b66c | ||
|
|
890ac4c86e | ||
| 9c6e974d3a | |||
| d1dc32d3bb | |||
| a6a0accd6a | |||
| 3b8b1f5492 | |||
| 0c80a08732 | |||
| 8b72e62621 | |||
| ff7a4b69cd | |||
| e7f9594381 | |||
| 0c0f219d02 | |||
| 9cf0ed84d9 | |||
| 2702edce22 | |||
| c33533fcd6 | |||
| 32eb047551 | |||
| d27d64e407 | |||
| fa6893aa98 | |||
| 77a8265b40 | |||
| c692f8d653 | |||
| 47096c9877 | |||
| 0104532e73 | |||
| 19e7e48190 | |||
| 7ef3ffe62c | |||
| 1853ef6452 | |||
| 34ef9490a4 | |||
| f3bc3da9bb | |||
| f797234abd | |||
| c95a4286fb | |||
| b72f281651 | |||
| 98912c5b03 | |||
| 28876fa1d2 | |||
| 5f46d26037 | |||
| 57433c7e75 | |||
| 9e7d7b88ac | |||
| bd88eaa3c8 | |||
| cd4cbec7e0 | |||
| 144f6e7c86 | |||
| ef8f7cbeaa | |||
|
|
507da3165f | ||
|
|
f41792a0fa | ||
|
|
e16a02453e | ||
|
|
b46d7d85c6 | ||
|
|
e5543f408a | ||
|
|
b239f564b2 | ||
| e6a2731d7e | |||
| bb2db662d5 | |||
| 84aade8c89 | |||
| 2e77fe85e4 | |||
| 64ea940f30 | |||
| a5ef8b5c46 | |||
| 2451e1b046 | |||
| 63288a2255 | |||
| 4657e766f1 | |||
| d3f00cfbb4 | |||
| 773784e1a3 | |||
| 56045f8f55 | |||
| 1ba4c72e11 | |||
| 1efa41c112 | |||
| b925ad78e8 | |||
| 0f638f1d70 | |||
| 5c59b27808 | |||
| d66ce89c7a | |||
| 78711b9baf | |||
| 51424eeefc | |||
| 45a637f545 | |||
| 1b1603c8f0 | |||
| 86fcc286e5 | |||
| 5992a54ac3 | |||
| 44853ab6f6 | |||
| c06a459b67 | |||
| 370cb3f74c | |||
| ead68a25cf | |||
| 392458ad99 | |||
| bd05eab467 | |||
| f39f201b70 | |||
| fee9120bc2 | |||
| f32be5ea1c | |||
| 1a7a1c4de8 | |||
| a628f9044f | |||
| 0c853354eb | |||
| 476eb331c2 | |||
| 93587645cd | |||
| 42f26089e1 | |||
| 75e87a7cc8 | |||
| 0eef6bca12 | |||
| ac50ca2117 | |||
| 5442d52ab1 | |||
| 227d035f01 | |||
| f1da4ca774 |
20
bootstrap/argocd/application-monitoring-extras.yaml
Normal file
20
bootstrap/argocd/application-monitoring-extras.yaml
Normal file
@@ -0,0 +1,20 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: monitoring-extras
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://gitea.mpabi.pl/trade/trade-deploy.git
|
||||
targetRevision: main
|
||||
path: kustomize/infra/monitoring-extras
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: monitoring
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
50
bootstrap/argocd/application-monitoring-stack.yaml
Normal file
50
bootstrap/argocd/application-monitoring-stack.yaml
Normal file
@@ -0,0 +1,50 @@
|
||||
apiVersion: argoproj.io/v1alpha1
|
||||
kind: Application
|
||||
metadata:
|
||||
name: monitoring-stack
|
||||
namespace: argocd
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://prometheus-community.github.io/helm-charts
|
||||
chart: kube-prometheus-stack
|
||||
targetRevision: 81.6.9
|
||||
helm:
|
||||
skipCrds: true
|
||||
values: |
|
||||
grafana:
|
||||
enabled: true
|
||||
prometheus:
|
||||
prometheusSpec:
|
||||
scrapeInterval: 15s
|
||||
evaluationInterval: 15s
|
||||
additionalScrapeConfigs:
|
||||
- job_name: mpabi-yellowstone-geyser
|
||||
metrics_path: /metrics
|
||||
scrape_interval: 10s
|
||||
fallback_scrape_protocol: PrometheusText0.0.4
|
||||
static_configs:
|
||||
- targets:
|
||||
- 10.66.66.1:8999
|
||||
- job_name: mpabi-node-exporter
|
||||
metrics_path: /metrics
|
||||
scrape_interval: 15s
|
||||
static_configs:
|
||||
- targets:
|
||||
- 10.66.66.1:9100
|
||||
prometheusOperator:
|
||||
admissionWebhooks:
|
||||
enabled: false
|
||||
patch:
|
||||
enabled: false
|
||||
destination:
|
||||
server: https://kubernetes.default.svc
|
||||
namespace: monitoring
|
||||
syncPolicy:
|
||||
automated:
|
||||
prune: true
|
||||
selfHeal: true
|
||||
syncOptions:
|
||||
- CreateNamespace=true
|
||||
- Replace=true
|
||||
- ServerSideApply=true
|
||||
@@ -6,7 +6,7 @@ metadata:
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://rv32i.pl/trade/trade-deploy.git
|
||||
repoURL: https://gitea.mpabi.pl/trade/trade-deploy.git
|
||||
targetRevision: main
|
||||
path: kustomize/infra/portainer
|
||||
destination:
|
||||
|
||||
@@ -6,7 +6,7 @@ metadata:
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://rv32i.pl/trade/trade-deploy.git
|
||||
repoURL: https://gitea.mpabi.pl/trade/trade-deploy.git
|
||||
targetRevision: main
|
||||
path: kustomize/overlays/prod
|
||||
destination:
|
||||
|
||||
@@ -6,7 +6,7 @@ metadata:
|
||||
spec:
|
||||
project: default
|
||||
source:
|
||||
repoURL: https://rv32i.pl/trade/trade-deploy.git
|
||||
repoURL: https://gitea.mpabi.pl/trade/trade-deploy.git
|
||||
targetRevision: main
|
||||
path: kustomize/overlays/staging
|
||||
destination:
|
||||
|
||||
@@ -1,54 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: trade-api
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: trade-api
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: trade-api
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: gitea-registry
|
||||
containers:
|
||||
- name: api
|
||||
image: rv32i.pl/trade/trade-api:k3s-20260106013603
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8787
|
||||
env:
|
||||
- name: PORT
|
||||
value: "8787"
|
||||
- name: APP_VERSION
|
||||
value: "staging"
|
||||
- name: HASURA_GRAPHQL_URL
|
||||
value: "http://hasura:8080/v1/graphql"
|
||||
- name: HASURA_ADMIN_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-hasura
|
||||
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||
- name: API_ADMIN_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-api
|
||||
key: API_ADMIN_SECRET
|
||||
- name: CORS_ORIGIN
|
||||
value: "*"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
initialDelaySeconds: 2
|
||||
periodSeconds: 5
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
@@ -1,12 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: trade-api
|
||||
spec:
|
||||
type: ClusterIP
|
||||
selector:
|
||||
app.kubernetes.io/name: trade-api
|
||||
ports:
|
||||
- name: http
|
||||
port: 8787
|
||||
targetPort: http
|
||||
48
kustomize/base/candles-cache-worker/deployment.yaml
Normal file
48
kustomize/base/candles-cache-worker/deployment.yaml
Normal file
@@ -0,0 +1,48 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: candles-cache-worker
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "6"
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: candles-cache-worker
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: candles-cache-worker
|
||||
spec:
|
||||
containers:
|
||||
- name: worker
|
||||
image: node:20-slim
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: HASURA_GRAPHQL_URL
|
||||
value: http://hasura:8080/v1/graphql
|
||||
- name: HASURA_ADMIN_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-hasura
|
||||
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||
- name: CANDLES_SYMBOLS
|
||||
value: SOL-PERP,PUMP-PERP
|
||||
- name: CANDLES_SOURCES
|
||||
value: any
|
||||
- name: CANDLES_TFS
|
||||
value: 1s,3s,5s,15s,30s,1m,3m,5m,15m,30m,1h,4h,12h,1d
|
||||
- name: CANDLES_TARGET_POINTS
|
||||
value: "1024"
|
||||
- name: CANDLES_POLL_MS
|
||||
value: "5000"
|
||||
command: ["node", "/app/worker.mjs"]
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /app/worker.mjs
|
||||
subPath: worker.mjs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: candles-cache-worker-script
|
||||
431
kustomize/base/candles-cache-worker/worker.mjs
Normal file
431
kustomize/base/candles-cache-worker/worker.mjs
Normal file
@@ -0,0 +1,431 @@
|
||||
import fs from 'node:fs';
|
||||
import process from 'node:process';
|
||||
import { setTimeout as sleep } from 'node:timers/promises';
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
try {
|
||||
const raw = fs.readFileSync(filePath, 'utf8');
|
||||
return JSON.parse(raw);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function getIsoNow() {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function clampInt(value, min, max, fallback) {
|
||||
const n = Number.parseInt(String(value ?? ''), 10);
|
||||
if (!Number.isInteger(n)) return fallback;
|
||||
return Math.min(max, Math.max(min, n));
|
||||
}
|
||||
|
||||
function envList(name, fallbackCsv) {
|
||||
const raw = process.env[name] ?? fallbackCsv;
|
||||
return String(raw)
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function parseTimeframeToSeconds(tf) {
|
||||
const v = String(tf || '').trim().toLowerCase();
|
||||
if (!v) return 60;
|
||||
const m = v.match(/^(\d+)(s|m|h|d)$/);
|
||||
if (!m) throw new Error('invalid_tf');
|
||||
const num = Number.parseInt(m[1], 10);
|
||||
if (!Number.isInteger(num) || num <= 0) throw new Error('invalid_tf');
|
||||
const unit = m[2];
|
||||
const mult = unit === 's' ? 1 : unit === 'm' ? 60 : unit === 'h' ? 3600 : 86400;
|
||||
return num * mult;
|
||||
}
|
||||
|
||||
function sqlLit(value) {
|
||||
const s = String(value ?? '');
|
||||
return `'${s.replace(/'/g, "''")}'`;
|
||||
}
|
||||
|
||||
function resolveHasuraBaseUrl(graphqlUrlOrBase) {
|
||||
const u = String(graphqlUrlOrBase || '').trim();
|
||||
if (!u) return 'http://hasura:8080';
|
||||
// common case: http://hasura:8080/v1/graphql
|
||||
if (u.endsWith('/v1/graphql')) return u.slice(0, -'/v1/graphql'.length);
|
||||
return u.replace(/\/$/, '');
|
||||
}
|
||||
|
||||
function resolveConfig() {
|
||||
const tokensPath =
|
||||
process.env.HASURA_TOKENS_FILE ||
|
||||
process.env.TOKENS_FILE ||
|
||||
process.env.HASURA_CONFIG_FILE ||
|
||||
'/app/tokens/hasura.json';
|
||||
const tokens = readJsonFile(tokensPath) || {};
|
||||
|
||||
const graphqlUrl =
|
||||
process.env.HASURA_GRAPHQL_URL ||
|
||||
tokens.graphqlUrl ||
|
||||
tokens.apiUrl ||
|
||||
'http://hasura:8080/v1/graphql';
|
||||
const hasuraBaseUrl = resolveHasuraBaseUrl(graphqlUrl);
|
||||
|
||||
const hasuraAdminSecret =
|
||||
process.env.HASURA_ADMIN_SECRET ||
|
||||
process.env.HASURA_GRAPHQL_ADMIN_SECRET ||
|
||||
tokens.adminSecret ||
|
||||
tokens.hasuraAdminSecret;
|
||||
|
||||
if (!hasuraAdminSecret) throw new Error('Missing HASURA_ADMIN_SECRET (required for /v2/query run_sql)');
|
||||
|
||||
const symbols = envList('CANDLES_SYMBOLS', 'SOL-PERP,PUMP-PERP');
|
||||
|
||||
// sources: include "any" (sourceKey='') by default.
|
||||
const rawSources = envList('CANDLES_SOURCES', 'any');
|
||||
const sources = [];
|
||||
for (const s of rawSources) {
|
||||
const v = String(s).trim();
|
||||
if (!v) continue;
|
||||
if (v.toLowerCase() === 'any') sources.push('');
|
||||
else sources.push(v);
|
||||
}
|
||||
if (!sources.includes('')) sources.unshift('');
|
||||
|
||||
const tfList = envList('CANDLES_TFS', '1s,3s,5s,15s,30s,1m,3m,5m,15m,30m,1h,4h,12h,1d');
|
||||
const bucketSecondsList = tfList
|
||||
.map((t) => {
|
||||
try {
|
||||
return parseTimeframeToSeconds(t);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})
|
||||
.filter((n) => n != null)
|
||||
.map((n) => n)
|
||||
.filter((n, idx, arr) => arr.findIndex((x) => x === n) === idx)
|
||||
.sort((a, b) => a - b);
|
||||
|
||||
const targetPoints = clampInt(process.env.CANDLES_TARGET_POINTS, 10, 100_000, 1024);
|
||||
// legacy: kept for compatibility; if set, used as a minimum warmup window (days).
|
||||
const backfillDays = clampInt(process.env.CANDLES_BACKFILL_DAYS, 0, 3650, 0);
|
||||
const pollMs = clampInt(process.env.CANDLES_POLL_MS, 250, 60_000, 5000);
|
||||
|
||||
return { hasuraBaseUrl, hasuraAdminSecret, symbols, sources, bucketSecondsList, targetPoints, backfillDays, pollMs };
|
||||
}
|
||||
|
||||
async function hasuraRunSql(cfg, sql, { readOnly } = { readOnly: false }) {
|
||||
const url = `${cfg.hasuraBaseUrl}/v2/query`;
|
||||
const body = {
|
||||
type: 'run_sql',
|
||||
args: {
|
||||
source: 'default',
|
||||
sql,
|
||||
read_only: Boolean(readOnly),
|
||||
},
|
||||
};
|
||||
|
||||
const res = await fetch(url, {
|
||||
method: 'POST',
|
||||
headers: { 'content-type': 'application/json', 'x-hasura-admin-secret': cfg.hasuraAdminSecret },
|
||||
body: JSON.stringify(body),
|
||||
signal: AbortSignal.timeout(60_000),
|
||||
});
|
||||
const text = await res.text();
|
||||
if (!res.ok) throw new Error(`Hasura run_sql HTTP ${res.status}: ${text}`);
|
||||
return JSON.parse(text);
|
||||
}
|
||||
|
||||
function chunkSecondsForBucket(bucketSeconds) {
|
||||
if (bucketSeconds <= 5) return 15 * 60;
|
||||
if (bucketSeconds <= 60) return 60 * 60;
|
||||
if (bucketSeconds <= 300) return 6 * 60 * 60;
|
||||
if (bucketSeconds <= 3600) return 24 * 60 * 60;
|
||||
return 7 * 24 * 60 * 60;
|
||||
}
|
||||
|
||||
function sqlUpsertCandlesFromTicks({ symbol, sourceKey, bucketSeconds, fromIso, toIso }) {
|
||||
return `
|
||||
WITH chosen AS (
|
||||
SELECT source AS chosen_source
|
||||
FROM public.drift_ticks
|
||||
WHERE symbol = ${sqlLit(symbol)}
|
||||
ORDER BY ts DESC
|
||||
LIMIT 1
|
||||
),
|
||||
base AS (
|
||||
SELECT
|
||||
time_bucket(make_interval(secs => ${bucketSeconds}), ts) AS bucket,
|
||||
ts,
|
||||
COALESCE(mark_price, oracle_price) AS px,
|
||||
oracle_price AS oracle_px
|
||||
FROM public.drift_ticks
|
||||
WHERE symbol = ${sqlLit(symbol)}
|
||||
AND ts >= ${sqlLit(fromIso)}::timestamptz
|
||||
AND ts < ${sqlLit(toIso)}::timestamptz
|
||||
AND (
|
||||
${sqlLit(sourceKey)} <> '' AND source = ${sqlLit(sourceKey)}
|
||||
OR ${sqlLit(sourceKey)} = '' AND source = COALESCE((SELECT chosen_source FROM chosen), source)
|
||||
)
|
||||
),
|
||||
agg AS (
|
||||
SELECT
|
||||
bucket,
|
||||
(array_agg(px ORDER BY ts ASC))[1] AS open,
|
||||
max(px) AS high,
|
||||
min(px) AS low,
|
||||
(array_agg(px ORDER BY ts DESC))[1] AS close,
|
||||
(array_agg(oracle_px ORDER BY ts ASC))[1] AS oracle_open,
|
||||
max(oracle_px) AS oracle_high,
|
||||
min(oracle_px) AS oracle_low,
|
||||
(array_agg(oracle_px ORDER BY ts DESC))[1] AS oracle_close,
|
||||
count(*)::bigint AS ticks
|
||||
FROM base
|
||||
GROUP BY bucket
|
||||
)
|
||||
INSERT INTO public.drift_candles_cache
|
||||
(
|
||||
bucket,
|
||||
bucket_seconds,
|
||||
symbol,
|
||||
source,
|
||||
open,
|
||||
high,
|
||||
low,
|
||||
close,
|
||||
oracle_open,
|
||||
oracle_high,
|
||||
oracle_low,
|
||||
oracle_close,
|
||||
ticks,
|
||||
updated_at
|
||||
)
|
||||
SELECT
|
||||
bucket,
|
||||
${bucketSeconds},
|
||||
${sqlLit(symbol)},
|
||||
${sqlLit(sourceKey)},
|
||||
open,
|
||||
high,
|
||||
low,
|
||||
close,
|
||||
oracle_open,
|
||||
oracle_high,
|
||||
oracle_low,
|
||||
oracle_close,
|
||||
ticks,
|
||||
now()
|
||||
FROM agg
|
||||
ON CONFLICT (bucket, bucket_seconds, symbol, source) DO UPDATE SET
|
||||
open = EXCLUDED.open,
|
||||
high = EXCLUDED.high,
|
||||
low = EXCLUDED.low,
|
||||
close = EXCLUDED.close,
|
||||
oracle_open = EXCLUDED.oracle_open,
|
||||
oracle_high = EXCLUDED.oracle_high,
|
||||
oracle_low = EXCLUDED.oracle_low,
|
||||
oracle_close = EXCLUDED.oracle_close,
|
||||
ticks = EXCLUDED.ticks,
|
||||
updated_at = now();
|
||||
`;
|
||||
}
|
||||
|
||||
function sqlDeleteOlderCandles({ symbol, sourceKey, bucketSeconds, cutoffIso }) {
|
||||
return `
|
||||
DELETE FROM public.drift_candles_cache
|
||||
WHERE symbol = ${sqlLit(symbol)}
|
||||
AND source = ${sqlLit(sourceKey)}
|
||||
AND bucket_seconds = ${bucketSeconds}
|
||||
AND bucket < ${sqlLit(cutoffIso)}::timestamptz;
|
||||
`;
|
||||
}
|
||||
|
||||
async function getTickRange(cfg, { symbol, sourceKey }) {
|
||||
const sql =
|
||||
String(sourceKey) === ''
|
||||
? `
|
||||
WITH chosen AS (
|
||||
SELECT source AS chosen_source
|
||||
FROM public.drift_ticks
|
||||
WHERE symbol=${sqlLit(symbol)}
|
||||
ORDER BY ts DESC
|
||||
LIMIT 1
|
||||
)
|
||||
SELECT min(ts) AS min_ts, max(ts) AS max_ts
|
||||
FROM public.drift_ticks
|
||||
WHERE symbol=${sqlLit(symbol)}
|
||||
AND source = COALESCE((SELECT chosen_source FROM chosen), source);
|
||||
`
|
||||
: `
|
||||
SELECT min(ts) AS min_ts, max(ts) AS max_ts
|
||||
FROM public.drift_ticks
|
||||
WHERE symbol=${sqlLit(symbol)}
|
||||
AND source = ${sqlLit(sourceKey)};
|
||||
`;
|
||||
const out = await hasuraRunSql(cfg, sql, { readOnly: true });
|
||||
const row = Array.isArray(out?.result) && out.result.length >= 2 ? out.result[1] : null;
|
||||
if (!row) return { minTs: null, maxTs: null };
|
||||
const minTs = row[0] ? String(row[0]).trim() : null;
|
||||
const maxTs = row[1] ? String(row[1]).trim() : null;
|
||||
return { minTs: minTs && minTs.length ? minTs : null, maxTs: maxTs && maxTs.length ? maxTs : null };
|
||||
}
|
||||
|
||||
function desiredFromIso({ minTsIso, maxTsIso, bucketSeconds, targetPoints, backfillDays }) {
|
||||
const endMs = Date.parse(maxTsIso);
|
||||
const minMs = minTsIso ? Date.parse(minTsIso) : null;
|
||||
const wantSpanMs = targetPoints * bucketSeconds * 1000;
|
||||
const wantFromMs = endMs - wantSpanMs;
|
||||
const minFromMs = backfillDays > 0 ? endMs - backfillDays * 24 * 60 * 60 * 1000 : null;
|
||||
let fromMs = wantFromMs;
|
||||
if (minFromMs != null) fromMs = Math.min(fromMs, minFromMs);
|
||||
if (minMs != null) fromMs = Math.max(fromMs, minMs);
|
||||
return new Date(Math.max(0, fromMs)).toISOString();
|
||||
}
|
||||
|
||||
function safetyWindowSeconds(bucketSeconds) {
|
||||
if (bucketSeconds <= 60) return 10 * 60;
|
||||
if (bucketSeconds <= 300) return 60 * 60;
|
||||
if (bucketSeconds <= 3600) return 6 * 60 * 60;
|
||||
if (bucketSeconds <= 14_400) return 24 * 60 * 60;
|
||||
return 2 * 24 * 60 * 60;
|
||||
}
|
||||
|
||||
async function backfillBucket(cfg, { symbol, sourceKey, bucketSeconds, fromIso, toIso }) {
|
||||
const chunk = chunkSecondsForBucket(bucketSeconds);
|
||||
for (let t = Date.parse(fromIso); t < Date.parse(toIso); t += chunk * 1000) {
|
||||
const a = new Date(t).toISOString();
|
||||
const b = new Date(Math.min(Date.parse(toIso), t + chunk * 1000)).toISOString();
|
||||
await hasuraRunSql(cfg, sqlUpsertCandlesFromTicks({ symbol, sourceKey, bucketSeconds, fromIso: a, toIso: b }));
|
||||
}
|
||||
}
|
||||
|
||||
async function getMaxBucket(cfg, { symbol, sourceKey, bucketSeconds }) {
|
||||
const sql = `
|
||||
SELECT max(bucket) AS max_bucket
|
||||
FROM public.drift_candles_cache
|
||||
WHERE symbol=${sqlLit(symbol)} AND source=${sqlLit(sourceKey)} AND bucket_seconds=${bucketSeconds};
|
||||
`;
|
||||
const out = await hasuraRunSql(cfg, sql, { readOnly: true });
|
||||
// Hasura returns {result_type, result:[[col...],[row...]]}
|
||||
const row = Array.isArray(out?.result) && out.result.length >= 2 ? out.result[1] : null;
|
||||
const v = row && row[0] ? String(row[0]) : null;
|
||||
return v && v.trim() ? v.trim() : null;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const cfg = resolveConfig();
|
||||
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
service: 'candles-cache-worker',
|
||||
startedAt: getIsoNow(),
|
||||
hasuraBaseUrl: cfg.hasuraBaseUrl,
|
||||
symbols: cfg.symbols,
|
||||
sources: cfg.sources.map((s) => (s ? s : '(any)')),
|
||||
bucketSecondsList: cfg.bucketSecondsList,
|
||||
targetPoints: cfg.targetPoints,
|
||||
backfillDays: cfg.backfillDays,
|
||||
pollMs: cfg.pollMs,
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
|
||||
// Backfill to warm cache: for each timeframe keep ~targetPoints candles (or "as much as we have").
|
||||
for (const symbol of cfg.symbols) {
|
||||
for (const sourceKey of cfg.sources) {
|
||||
const range = await getTickRange(cfg, { symbol, sourceKey });
|
||||
if (!range.maxTs) continue;
|
||||
const toIso = new Date(Date.parse(range.maxTs)).toISOString();
|
||||
for (const bs of cfg.bucketSecondsList) {
|
||||
const fromIso = desiredFromIso({
|
||||
minTsIso: range.minTs,
|
||||
maxTsIso: toIso,
|
||||
bucketSeconds: bs,
|
||||
targetPoints: cfg.targetPoints,
|
||||
backfillDays: cfg.backfillDays,
|
||||
});
|
||||
|
||||
console.log(
|
||||
`[candles-cache-worker] warmup symbol=${symbol} source=${sourceKey || '(any)'} bs=${bs}s from=${fromIso} to=${toIso} points=${cfg.targetPoints}`
|
||||
);
|
||||
try {
|
||||
await backfillBucket(cfg, { symbol, sourceKey, bucketSeconds: bs, fromIso, toIso });
|
||||
// Enforce max window for this bucket (derived data; safe to prune).
|
||||
await hasuraRunSql(cfg, sqlDeleteOlderCandles({ symbol, sourceKey, bucketSeconds: bs, cutoffIso: fromIso }));
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[candles-cache-worker] warmup failed (${symbol}/${sourceKey || 'any'}/${bs}s): ${String(err?.message || err)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Prime last buckets.
|
||||
const last = new Map(); // key -> iso bucket
|
||||
for (const symbol of cfg.symbols) {
|
||||
for (const sourceKey of cfg.sources) {
|
||||
for (const bs of cfg.bucketSecondsList) {
|
||||
const k = `${symbol}::${sourceKey}::${bs}`;
|
||||
try {
|
||||
const maxBucket = await getMaxBucket(cfg, { symbol, sourceKey, bucketSeconds: bs });
|
||||
if (maxBucket) last.set(k, maxBucket);
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
while (true) {
|
||||
const loopNow = Date.now();
|
||||
const loopIso = new Date(loopNow).toISOString();
|
||||
const pruneEveryMs = 60_000;
|
||||
const lastPruneAt = last.__pruneAt || new Map();
|
||||
// stash on the Map to keep closure-local without introducing a new outer var
|
||||
last.__pruneAt = lastPruneAt;
|
||||
|
||||
for (const symbol of cfg.symbols) {
|
||||
for (const sourceKey of cfg.sources) {
|
||||
for (const bs of cfg.bucketSecondsList) {
|
||||
const k = `${symbol}::${sourceKey}::${bs}`;
|
||||
const prev = last.get(k);
|
||||
const safety = safetyWindowSeconds(bs);
|
||||
const fromMs = prev ? Date.parse(prev) - safety * 1000 : loopNow - safety * 1000;
|
||||
const fromIso2 = new Date(Math.max(0, fromMs)).toISOString();
|
||||
try {
|
||||
await hasuraRunSql(
|
||||
cfg,
|
||||
sqlUpsertCandlesFromTicks({ symbol, sourceKey, bucketSeconds: bs, fromIso: fromIso2, toIso: loopIso })
|
||||
);
|
||||
// best-effort: move last pointer close to now (actual max will lag by at most one bucket)
|
||||
last.set(k, loopIso);
|
||||
|
||||
const prevPrune = lastPruneAt.get(k) || 0;
|
||||
if (loopNow - prevPrune >= pruneEveryMs) {
|
||||
const keepSeconds = Math.max(cfg.targetPoints * bs, (cfg.backfillDays > 0 ? cfg.backfillDays * 86400 : 0));
|
||||
const cutoffIso = new Date(Math.max(0, loopNow - keepSeconds * 1000)).toISOString();
|
||||
try {
|
||||
await hasuraRunSql(cfg, sqlDeleteOlderCandles({ symbol, sourceKey, bucketSeconds: bs, cutoffIso }));
|
||||
} finally {
|
||||
lastPruneAt.set(k, loopNow);
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
`[candles-cache-worker] update failed (${symbol}/${sourceKey || 'any'}/${bs}s): ${String(err?.message || err)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(cfg.pollMs);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(String(err?.stack || err));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
44
kustomize/base/dlob-ts-archiver/deployment-drift.yaml
Normal file
44
kustomize/base/dlob-ts-archiver/deployment-drift.yaml
Normal file
@@ -0,0 +1,44 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-ts-archiver-drift
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "6"
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: dlob-ts-archiver-drift
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: dlob-ts-archiver-drift
|
||||
spec:
|
||||
containers:
|
||||
- name: worker
|
||||
image: node:20-slim
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: HASURA_GRAPHQL_URL
|
||||
value: http://hasura:8080/v1/graphql
|
||||
- name: HASURA_ADMIN_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-hasura
|
||||
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||
- name: DLOB_SOURCE
|
||||
value: drift
|
||||
- name: DLOB_MARKETS
|
||||
value: SOL-PERP,DOGE-PERP,JUP-PERP
|
||||
- name: DLOB_TS_POLL_MS
|
||||
value: "1000"
|
||||
command: ["node", "/app/worker.mjs"]
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /app/worker.mjs
|
||||
subPath: worker.mjs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: dlob-ts-archiver-script
|
||||
44
kustomize/base/dlob-ts-archiver/deployment.yaml
Normal file
44
kustomize/base/dlob-ts-archiver/deployment.yaml
Normal file
@@ -0,0 +1,44 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-ts-archiver
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "6"
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: dlob-ts-archiver
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: dlob-ts-archiver
|
||||
spec:
|
||||
containers:
|
||||
- name: worker
|
||||
image: node:20-slim
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: HASURA_GRAPHQL_URL
|
||||
value: http://hasura:8080/v1/graphql
|
||||
- name: HASURA_ADMIN_SECRET
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-hasura
|
||||
key: HASURA_GRAPHQL_ADMIN_SECRET
|
||||
- name: DLOB_SOURCE
|
||||
value: mevnode
|
||||
- name: DLOB_MARKETS
|
||||
value: SOL-PERP,DOGE-PERP,JUP-PERP
|
||||
- name: DLOB_TS_POLL_MS
|
||||
value: "1000"
|
||||
command: ["node", "/app/worker.mjs"]
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /app/worker.mjs
|
||||
subPath: worker.mjs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: dlob-ts-archiver-script
|
||||
254
kustomize/base/dlob-ts-archiver/worker.mjs
Normal file
254
kustomize/base/dlob-ts-archiver/worker.mjs
Normal file
@@ -0,0 +1,254 @@
|
||||
import fs from 'node:fs';
|
||||
import process from 'node:process';
|
||||
import { setTimeout as sleep } from 'node:timers/promises';
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
try {
|
||||
const raw = fs.readFileSync(filePath, 'utf8');
|
||||
return JSON.parse(raw);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function getIsoNow() {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
function clampInt(value, min, max, fallback) {
|
||||
const n = Number.parseInt(String(value ?? ''), 10);
|
||||
if (!Number.isInteger(n)) return fallback;
|
||||
return Math.min(max, Math.max(min, n));
|
||||
}
|
||||
|
||||
function envList(name, fallbackCsv) {
|
||||
const raw = process.env[name] ?? fallbackCsv;
|
||||
return String(raw)
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function resolveConfig() {
|
||||
const tokensPath =
|
||||
process.env.HASURA_TOKENS_FILE ||
|
||||
process.env.TOKENS_FILE ||
|
||||
process.env.HASURA_CONFIG_FILE ||
|
||||
'/app/tokens/hasura.json';
|
||||
const tokens = readJsonFile(tokensPath) || {};
|
||||
|
||||
const hasuraUrl =
|
||||
process.env.HASURA_GRAPHQL_URL ||
|
||||
tokens.graphqlUrl ||
|
||||
tokens.apiUrl ||
|
||||
'http://hasura:8080/v1/graphql';
|
||||
const hasuraAdminSecret =
|
||||
process.env.HASURA_ADMIN_SECRET ||
|
||||
process.env.HASURA_GRAPHQL_ADMIN_SECRET ||
|
||||
tokens.adminSecret ||
|
||||
tokens.hasuraAdminSecret;
|
||||
const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined;
|
||||
|
||||
const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode';
|
||||
const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP');
|
||||
const pollMs = clampInt(process.env.DLOB_TS_POLL_MS, 500, 60_000, 1000);
|
||||
|
||||
return { hasuraUrl, hasuraAdminSecret, hasuraAuthToken, dlobSource, markets, pollMs };
|
||||
}
|
||||
|
||||
async function graphqlRequest(cfg, query, variables) {
|
||||
const headers = { 'content-type': 'application/json' };
|
||||
if (cfg.hasuraAuthToken) {
|
||||
headers.authorization = `Bearer ${cfg.hasuraAuthToken}`;
|
||||
} else if (cfg.hasuraAdminSecret) {
|
||||
headers['x-hasura-admin-secret'] = cfg.hasuraAdminSecret;
|
||||
} else {
|
||||
throw new Error('Missing Hasura auth (set HASURA_AUTH_TOKEN or HASURA_ADMIN_SECRET or mount tokens/hasura.json)');
|
||||
}
|
||||
|
||||
const res = await fetch(cfg.hasuraUrl, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: JSON.stringify({ query, variables }),
|
||||
signal: AbortSignal.timeout(15_000),
|
||||
});
|
||||
const text = await res.text();
|
||||
if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`);
|
||||
const json = JSON.parse(text);
|
||||
if (json.errors?.length) {
|
||||
throw new Error(json.errors.map((e) => e.message).join(' | '));
|
||||
}
|
||||
return json.data;
|
||||
}
|
||||
|
||||
function mapBigint(v) {
|
||||
if (v == null) return null;
|
||||
if (typeof v === 'number') return Number.isFinite(v) ? String(Math.trunc(v)) : null;
|
||||
if (typeof v === 'string') return v.trim() || null;
|
||||
return null;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const cfg = resolveConfig();
|
||||
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
service: 'dlob-ts-archiver',
|
||||
startedAt: getIsoNow(),
|
||||
hasuraUrl: cfg.hasuraUrl,
|
||||
hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none',
|
||||
dlobSource: cfg.dlobSource,
|
||||
markets: cfg.markets,
|
||||
pollMs: cfg.pollMs,
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
|
||||
while (true) {
|
||||
const now = getIsoNow();
|
||||
|
||||
try {
|
||||
const query = `
|
||||
query Latest($source: String!, $markets: [String!]!) {
|
||||
dlob_stats_latest(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
|
||||
market_name market_type market_index ts slot
|
||||
mark_price oracle_price best_bid_price best_ask_price mid_price
|
||||
spread_abs spread_bps depth_levels depth_bid_base depth_ask_base depth_bid_usd depth_ask_usd imbalance
|
||||
raw
|
||||
}
|
||||
dlob_depth_bps_latest(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
|
||||
market_name band_bps market_type market_index ts slot
|
||||
mid_price best_bid_price best_ask_price bid_base ask_base bid_usd ask_usd imbalance
|
||||
raw
|
||||
}
|
||||
dlob_slippage_latest(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
|
||||
market_name side size_usd market_type market_index ts slot
|
||||
mid_price vwap_price worst_price filled_usd filled_base impact_bps levels_consumed fill_pct
|
||||
raw
|
||||
}
|
||||
dlob_slippage_latest_v2(where: { source: { _eq: $source }, market_name: { _in: $markets } }) {
|
||||
market_name side size_usd market_type market_index ts slot
|
||||
mid_price vwap_price worst_price filled_usd filled_base impact_bps levels_consumed fill_pct
|
||||
raw
|
||||
}
|
||||
}
|
||||
`;
|
||||
|
||||
const data = await graphqlRequest(cfg, query, { source: cfg.dlobSource, markets: cfg.markets });
|
||||
|
||||
const statsRows = (data?.dlob_stats_latest || []).map((r) => ({
|
||||
ts: now,
|
||||
source: cfg.dlobSource,
|
||||
market_name: r.market_name,
|
||||
market_type: r.market_type,
|
||||
market_index: r.market_index ?? null,
|
||||
source_ts: mapBigint(r.ts),
|
||||
slot: mapBigint(r.slot),
|
||||
mark_price: r.mark_price ?? null,
|
||||
oracle_price: r.oracle_price ?? null,
|
||||
best_bid_price: r.best_bid_price ?? null,
|
||||
best_ask_price: r.best_ask_price ?? null,
|
||||
mid_price: r.mid_price ?? null,
|
||||
spread_abs: r.spread_abs ?? null,
|
||||
spread_bps: r.spread_bps ?? null,
|
||||
depth_levels: r.depth_levels ?? null,
|
||||
depth_bid_base: r.depth_bid_base ?? null,
|
||||
depth_ask_base: r.depth_ask_base ?? null,
|
||||
depth_bid_usd: r.depth_bid_usd ?? null,
|
||||
depth_ask_usd: r.depth_ask_usd ?? null,
|
||||
imbalance: r.imbalance ?? null,
|
||||
raw: r.raw ?? null,
|
||||
}));
|
||||
|
||||
const depthRows = (data?.dlob_depth_bps_latest || []).map((r) => ({
|
||||
ts: now,
|
||||
source: cfg.dlobSource,
|
||||
market_name: r.market_name,
|
||||
band_bps: r.band_bps,
|
||||
market_type: r.market_type,
|
||||
market_index: r.market_index ?? null,
|
||||
source_ts: mapBigint(r.ts),
|
||||
slot: mapBigint(r.slot),
|
||||
mid_price: r.mid_price ?? null,
|
||||
best_bid_price: r.best_bid_price ?? null,
|
||||
best_ask_price: r.best_ask_price ?? null,
|
||||
bid_base: r.bid_base ?? null,
|
||||
ask_base: r.ask_base ?? null,
|
||||
bid_usd: r.bid_usd ?? null,
|
||||
ask_usd: r.ask_usd ?? null,
|
||||
imbalance: r.imbalance ?? null,
|
||||
raw: r.raw ?? null,
|
||||
}));
|
||||
|
||||
const slippageRows = (data?.dlob_slippage_latest || []).map((r) => ({
|
||||
ts: now,
|
||||
source: cfg.dlobSource,
|
||||
market_name: r.market_name,
|
||||
side: r.side,
|
||||
size_usd: r.size_usd,
|
||||
market_type: r.market_type,
|
||||
market_index: r.market_index ?? null,
|
||||
source_ts: mapBigint(r.ts),
|
||||
slot: mapBigint(r.slot),
|
||||
mid_price: r.mid_price ?? null,
|
||||
vwap_price: r.vwap_price ?? null,
|
||||
worst_price: r.worst_price ?? null,
|
||||
filled_usd: r.filled_usd ?? null,
|
||||
filled_base: r.filled_base ?? null,
|
||||
impact_bps: r.impact_bps ?? null,
|
||||
levels_consumed: r.levels_consumed ?? null,
|
||||
fill_pct: r.fill_pct ?? null,
|
||||
raw: r.raw ?? null,
|
||||
}));
|
||||
|
||||
const slippageRowsV2 = (data?.dlob_slippage_latest_v2 || []).map((r) => ({
|
||||
ts: now,
|
||||
source: cfg.dlobSource,
|
||||
market_name: r.market_name,
|
||||
side: r.side,
|
||||
size_usd: r.size_usd,
|
||||
market_type: r.market_type,
|
||||
market_index: r.market_index ?? null,
|
||||
source_ts: mapBigint(r.ts),
|
||||
slot: mapBigint(r.slot),
|
||||
mid_price: r.mid_price ?? null,
|
||||
vwap_price: r.vwap_price ?? null,
|
||||
worst_price: r.worst_price ?? null,
|
||||
filled_usd: r.filled_usd ?? null,
|
||||
filled_base: r.filled_base ?? null,
|
||||
impact_bps: r.impact_bps ?? null,
|
||||
levels_consumed: r.levels_consumed ?? null,
|
||||
fill_pct: r.fill_pct ?? null,
|
||||
raw: r.raw ?? null,
|
||||
}));
|
||||
|
||||
const mutation = `
|
||||
mutation InsertTs(
|
||||
$stats: [dlob_stats_ts_insert_input!]!
|
||||
$depth: [dlob_depth_bps_ts_insert_input!]!
|
||||
$slip: [dlob_slippage_ts_insert_input!]!
|
||||
$slipV2: [dlob_slippage_ts_v2_insert_input!]!
|
||||
) {
|
||||
insert_dlob_stats_ts(objects: $stats) { affected_rows }
|
||||
insert_dlob_depth_bps_ts(objects: $depth) { affected_rows }
|
||||
insert_dlob_slippage_ts(objects: $slip) { affected_rows }
|
||||
insert_dlob_slippage_ts_v2(objects: $slipV2) { affected_rows }
|
||||
}
|
||||
`;
|
||||
|
||||
await graphqlRequest(cfg, mutation, { stats: statsRows, depth: depthRows, slip: slippageRows, slipV2: slippageRowsV2 });
|
||||
} catch (err) {
|
||||
console.error(`[dlob-ts-archiver] ${String(err?.message || err)}`);
|
||||
}
|
||||
|
||||
await sleep(cfg.pollMs);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(String(err?.stack || err));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
73
kustomize/base/dlob/publisher-deployment.yaml
Normal file
73
kustomize/base/dlob/publisher-deployment.yaml
Normal file
@@ -0,0 +1,73 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-publisher-hot
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "4"
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: dlob-publisher-hot
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: dlob-publisher-hot
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: gitea-registry
|
||||
containers:
|
||||
- name: publisher
|
||||
image: gitea.mpabi.pl/trade/trade-dlob-server:grpc-reconnect-20260319-023351
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
env:
|
||||
- name: RUNNING_LOCAL
|
||||
value: "true"
|
||||
- name: LOCAL_CACHE
|
||||
value: "true"
|
||||
- name: ENV
|
||||
value: mainnet-beta
|
||||
- name: USE_WEBSOCKET
|
||||
value: "true"
|
||||
- name: USE_ORDER_SUBSCRIBER
|
||||
value: "true"
|
||||
- name: DISABLE_GPA_REFRESH
|
||||
value: "true"
|
||||
- name: ELASTICACHE_HOST
|
||||
value: dlob-redis
|
||||
- name: ELASTICACHE_PORT
|
||||
value: "6379"
|
||||
- name: REDIS_CLIENT
|
||||
value: DLOB_HOT
|
||||
- name: PERP_MARKETS_TO_LOAD
|
||||
value: "0,7,24"
|
||||
- name: ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-dlob-rpc
|
||||
key: ENDPOINT
|
||||
- name: WS_ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-dlob-rpc
|
||||
key: WS_ENDPOINT
|
||||
command: ["node", "/lib/publishers/dlobPublisher.js"]
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /startup
|
||||
port: http
|
||||
initialDelaySeconds: 120
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 30
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 240
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 10
|
||||
79
kustomize/base/dlob/redis.yaml
Normal file
79
kustomize/base/dlob/redis.yaml
Normal file
@@ -0,0 +1,79 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: dlob-redis
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "3"
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: dlob-redis
|
||||
ports:
|
||||
- name: redis
|
||||
port: 6379
|
||||
targetPort: redis
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-redis
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "3"
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: dlob-redis
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: dlob-redis
|
||||
spec:
|
||||
containers:
|
||||
- name: redis
|
||||
image: redis:7-alpine
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: POD_IP
|
||||
valueFrom:
|
||||
fieldRef:
|
||||
fieldPath: status.podIP
|
||||
ports:
|
||||
- name: redis
|
||||
containerPort: 6379
|
||||
# DLOB redis client uses ioredis Cluster when RUNNING_LOCAL=true and LOCAL_CACHE=true.
|
||||
# We run a single-node Redis Cluster (no TLS) and assign all slots on startup.
|
||||
command: ["/bin/sh", "-lc"]
|
||||
args:
|
||||
- |
|
||||
exec redis-server \
|
||||
--save "" \
|
||||
--appendonly no \
|
||||
--protected-mode no \
|
||||
--bind 0.0.0.0 \
|
||||
--cluster-enabled yes \
|
||||
--cluster-config-file /data/nodes.conf \
|
||||
--cluster-node-timeout 5000 \
|
||||
--cluster-require-full-coverage no \
|
||||
--cluster-announce-ip "${POD_IP}" \
|
||||
--cluster-announce-port 6379 \
|
||||
--cluster-announce-bus-port 16379
|
||||
lifecycle:
|
||||
postStart:
|
||||
exec:
|
||||
command:
|
||||
- /bin/sh
|
||||
- -lc
|
||||
- |
|
||||
set -e
|
||||
for i in $(seq 1 60); do
|
||||
redis-cli -h 127.0.0.1 -p 6379 ping >/dev/null 2>&1 && break
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# If cluster is already initialized, do nothing.
|
||||
if redis-cli -h 127.0.0.1 -p 6379 cluster info 2>/dev/null | grep -q 'cluster_slots_assigned:16384'; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Redis 7+ supports CLUSTER ADDSLOTSRANGE.
|
||||
redis-cli -h 127.0.0.1 -p 6379 cluster addslotsrange 0 16383 || true
|
||||
@@ -16,7 +16,7 @@ spec:
|
||||
- name: gitea-registry
|
||||
containers:
|
||||
- name: frontend
|
||||
image: rv32i.pl/trade/trade-frontend:sha-e20a1f5
|
||||
image: gitea.mpabi.pl/trade/trade-frontend:pg-derived-phase1-20260319-030252
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
@@ -26,12 +26,8 @@ spec:
|
||||
value: "8081"
|
||||
- name: APP_VERSION
|
||||
value: "staging"
|
||||
- name: API_UPSTREAM
|
||||
value: "http://trade-api:8787"
|
||||
- name: BASIC_AUTH_FILE
|
||||
value: "/tokens/frontend.json"
|
||||
- name: API_READ_TOKEN_FILE
|
||||
value: "/tokens/read.json"
|
||||
volumeMounts:
|
||||
- name: tokens
|
||||
mountPath: /tokens
|
||||
|
||||
@@ -37,6 +37,8 @@ spec:
|
||||
value: "false"
|
||||
- name: HASURA_GRAPHQL_CORS_DOMAIN
|
||||
value: "http://localhost:5173,http://127.0.0.1:5173"
|
||||
- name: HASURA_GRAPHQL_UNAUTHORIZED_ROLE
|
||||
value: "public"
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /healthz
|
||||
|
||||
@@ -51,7 +51,7 @@ function isAlreadyExistsError(errText) {
|
||||
|
||||
function isMissingRelationError(errText) {
|
||||
const t = String(errText || '').toLowerCase();
|
||||
return t.includes('does not exist') || t.includes('not found') || t.includes('not tracked');
|
||||
return t.includes('does not exist') || t.includes('not found') || t.includes('not tracked') || t.includes('already untracked');
|
||||
}
|
||||
|
||||
function normalizeName(name) {
|
||||
@@ -89,12 +89,36 @@ async function main() {
|
||||
console.log(`[hasura-bootstrap] HASURA_URL=${HASURA_URL}`);
|
||||
await waitForHasura();
|
||||
|
||||
const PUBLIC_DLOB_SOURCE_HEADER = 'X-Hasura-Dlob-Source';
|
||||
|
||||
const apiTokensTable = { schema: 'public', name: 'api_tokens' };
|
||||
const botConfigTable = { schema: 'public', name: 'bot_config' };
|
||||
const botStateTable = { schema: 'public', name: 'bot_state' };
|
||||
const botEventsTable = { schema: 'public', name: 'bot_events' };
|
||||
const source = 'default';
|
||||
|
||||
const baseTicks = { schema: 'public', name: 'drift_ticks' };
|
||||
const dlobHotSnapshotLatestTable = { schema: 'public', name: 'dlob_hot_snapshot_latest' };
|
||||
const dlobHotDerivedLatestTable = { schema: 'public', name: 'dlob_hot_derived_latest' };
|
||||
const dlobAllDerivedLatestTable = { schema: 'public', name: 'dlob_all_derived_latest' };
|
||||
const candlesCacheTable = { schema: 'public', name: 'drift_candles_cache' };
|
||||
const dlobHotDerivedTsTable = { schema: 'public', name: 'dlob_hot_derived_ts' };
|
||||
const dlobAllDerivedTsTable = { schema: 'public', name: 'dlob_all_derived_ts' };
|
||||
const baseCandlesFn = { schema: 'public', name: 'get_drift_candles' };
|
||||
const candlesReturnTable = { schema: 'public', name: 'drift_candles' };
|
||||
const legacyDlobRelations = [
|
||||
{ schema: 'public', name: 'dlob_l2_latest' },
|
||||
{ schema: 'public', name: 'dlob_l2_latest_projection' },
|
||||
{ schema: 'public', name: 'dlob_stats_latest' },
|
||||
{ schema: 'public', name: 'dlob_stats_latest_projection' },
|
||||
{ schema: 'public', name: 'dlob_depth_bps_latest' },
|
||||
{ schema: 'public', name: 'dlob_slippage_latest' },
|
||||
{ schema: 'public', name: 'dlob_slippage_latest_v2' },
|
||||
{ schema: 'public', name: 'dlob_stats_ts' },
|
||||
{ schema: 'public', name: 'dlob_depth_bps_ts' },
|
||||
{ schema: 'public', name: 'dlob_slippage_ts' },
|
||||
{ schema: 'public', name: 'dlob_slippage_ts_v2' },
|
||||
];
|
||||
|
||||
const extraTicksName = normalizeName(TARGET_TICKS_TABLE);
|
||||
const extraCandlesName = normalizeName(TARGET_CANDLES_FUNCTION);
|
||||
@@ -161,8 +185,390 @@ async function main() {
|
||||
for (const t of tickTables) {
|
||||
await ensureTickTable(t);
|
||||
}
|
||||
|
||||
// Cached candles table (precomputed by worker; public read).
|
||||
await ensurePublicSelectTable(candlesCacheTable, [
|
||||
'bucket',
|
||||
'bucket_seconds',
|
||||
'symbol',
|
||||
'source',
|
||||
'open',
|
||||
'high',
|
||||
'low',
|
||||
'close',
|
||||
'oracle_close',
|
||||
'ticks',
|
||||
'updated_at',
|
||||
]);
|
||||
|
||||
const ensureDlobTable = async (table, columns, { publicFilter } = {}) => {
|
||||
await metadataIgnore({ type: 'pg_untrack_table', args: { source, table } });
|
||||
await metadata({ type: 'pg_track_table', args: { source, table } });
|
||||
|
||||
await metadataIgnore({ type: 'pg_drop_select_permission', args: { source, table, role: 'public' } });
|
||||
await metadata({
|
||||
type: 'pg_create_select_permission',
|
||||
args: {
|
||||
source,
|
||||
table,
|
||||
role: 'public',
|
||||
permission: {
|
||||
columns,
|
||||
filter: publicFilter || {},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await metadataIgnore({ type: 'pg_drop_insert_permission', args: { source, table, role: 'ingestor' } });
|
||||
await metadata({
|
||||
type: 'pg_create_insert_permission',
|
||||
args: {
|
||||
source,
|
||||
table,
|
||||
role: 'ingestor',
|
||||
permission: {
|
||||
check: {},
|
||||
set: {},
|
||||
columns,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await metadataIgnore({ type: 'pg_drop_update_permission', args: { source, table, role: 'ingestor' } });
|
||||
await metadata({
|
||||
type: 'pg_create_update_permission',
|
||||
args: {
|
||||
source,
|
||||
table,
|
||||
role: 'ingestor',
|
||||
permission: {
|
||||
filter: {},
|
||||
check: {},
|
||||
columns,
|
||||
},
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
async function ensurePublicSelectTable(table, columns, { publicFilter } = {}) {
|
||||
await metadataIgnore({ type: 'pg_untrack_table', args: { source, table } });
|
||||
await metadata({ type: 'pg_track_table', args: { source, table } });
|
||||
|
||||
await metadataIgnore({ type: 'pg_drop_select_permission', args: { source, table, role: 'public' } });
|
||||
await metadata({
|
||||
type: 'pg_create_select_permission',
|
||||
args: {
|
||||
source,
|
||||
table,
|
||||
role: 'public',
|
||||
permission: {
|
||||
columns,
|
||||
filter: publicFilter || {},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Computed/archived tables are written by workers using admin secret; keep ingestor off by default.
|
||||
await metadataIgnore({ type: 'pg_drop_insert_permission', args: { source, table, role: 'ingestor' } });
|
||||
await metadataIgnore({ type: 'pg_drop_update_permission', args: { source, table, role: 'ingestor' } });
|
||||
}
|
||||
|
||||
async function ensureProjectedPublicSelectTable(table, columns, { publicFilter, customName } = {}) {
|
||||
await metadataIgnore({ type: 'pg_untrack_table', args: { source, table, cascade: true } });
|
||||
const args = { source, table };
|
||||
if (customName) {
|
||||
args.configuration = { custom_name: customName };
|
||||
}
|
||||
await metadata({ type: 'pg_track_table', args });
|
||||
|
||||
await metadataIgnore({ type: 'pg_drop_select_permission', args: { source, table, role: 'public' } });
|
||||
await metadata({
|
||||
type: 'pg_create_select_permission',
|
||||
args: {
|
||||
source,
|
||||
table,
|
||||
role: 'public',
|
||||
permission: {
|
||||
columns,
|
||||
filter: publicFilter || {},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await metadataIgnore({ type: 'pg_drop_insert_permission', args: { source, table, role: 'ingestor' } });
|
||||
await metadataIgnore({ type: 'pg_drop_update_permission', args: { source, table, role: 'ingestor' } });
|
||||
}
|
||||
|
||||
const dlobPublicFilter = { source: { _eq: PUBLIC_DLOB_SOURCE_HEADER } };
|
||||
|
||||
for (const table of legacyDlobRelations) {
|
||||
await metadataIgnore({ type: 'pg_untrack_table', args: { source, table, cascade: true } });
|
||||
}
|
||||
|
||||
await ensurePublicSelectTable(dlobHotSnapshotLatestTable, [
|
||||
'source',
|
||||
'redis_key',
|
||||
'snapshot_kind',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'market_name',
|
||||
'is_indicative',
|
||||
'ts_ms',
|
||||
'slot',
|
||||
'market_slot',
|
||||
'payload_hash',
|
||||
'mark_price_raw',
|
||||
'oracle_price_raw',
|
||||
'best_bid_price_raw',
|
||||
'best_ask_price_raw',
|
||||
'spread_pct_raw',
|
||||
'spread_quote_raw',
|
||||
'oracle_data',
|
||||
'bids',
|
||||
'asks',
|
||||
'payload',
|
||||
'updated_at',
|
||||
]);
|
||||
|
||||
await ensurePublicSelectTable(dlobHotDerivedLatestTable, [
|
||||
'source',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'market_name',
|
||||
'is_indicative',
|
||||
'ts_ms',
|
||||
'slot',
|
||||
'market_slot',
|
||||
'mark_price',
|
||||
'oracle_price',
|
||||
'best_bid_price',
|
||||
'best_ask_price',
|
||||
'mid_price',
|
||||
'spread_quote',
|
||||
'spread_bps',
|
||||
'depth_levels',
|
||||
'bid_levels',
|
||||
'ask_levels',
|
||||
'top_bid_size',
|
||||
'top_ask_size',
|
||||
'top_bid_notional',
|
||||
'top_ask_notional',
|
||||
'depth_bid_base',
|
||||
'depth_ask_base',
|
||||
'depth_bid_quote',
|
||||
'depth_ask_quote',
|
||||
'imbalance',
|
||||
'bids_norm',
|
||||
'asks_norm',
|
||||
'raw_payload_hash',
|
||||
'updated_at',
|
||||
]);
|
||||
|
||||
await ensurePublicSelectTable(dlobAllDerivedLatestTable, [
|
||||
'source',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'market_name',
|
||||
'is_indicative',
|
||||
'ts_ms',
|
||||
'slot',
|
||||
'market_slot',
|
||||
'mark_price',
|
||||
'oracle_price',
|
||||
'best_bid_price',
|
||||
'best_ask_price',
|
||||
'mid_price',
|
||||
'spread_quote',
|
||||
'spread_bps',
|
||||
'depth_levels',
|
||||
'bid_levels',
|
||||
'ask_levels',
|
||||
'top_bid_size',
|
||||
'top_ask_size',
|
||||
'top_bid_notional',
|
||||
'top_ask_notional',
|
||||
'depth_bid_base',
|
||||
'depth_ask_base',
|
||||
'depth_bid_quote',
|
||||
'depth_ask_quote',
|
||||
'imbalance',
|
||||
'bids_norm',
|
||||
'asks_norm',
|
||||
'raw_payload_hash',
|
||||
'updated_at',
|
||||
]);
|
||||
|
||||
await ensurePublicSelectTable(dlobHotDerivedTsTable, [
|
||||
'event_ts',
|
||||
'id',
|
||||
'source',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'market_name',
|
||||
'is_indicative',
|
||||
'ts_ms',
|
||||
'slot',
|
||||
'market_slot',
|
||||
'mark_price',
|
||||
'oracle_price',
|
||||
'best_bid_price',
|
||||
'best_ask_price',
|
||||
'mid_price',
|
||||
'spread_quote',
|
||||
'spread_bps',
|
||||
'depth_levels',
|
||||
'bid_levels',
|
||||
'ask_levels',
|
||||
'top_bid_size',
|
||||
'top_ask_size',
|
||||
'top_bid_notional',
|
||||
'top_ask_notional',
|
||||
'depth_bid_base',
|
||||
'depth_ask_base',
|
||||
'depth_bid_quote',
|
||||
'depth_ask_quote',
|
||||
'imbalance',
|
||||
'bids_norm',
|
||||
'asks_norm',
|
||||
'raw_payload_hash',
|
||||
'inserted_at',
|
||||
]);
|
||||
|
||||
await ensurePublicSelectTable(dlobAllDerivedTsTable, [
|
||||
'event_ts',
|
||||
'id',
|
||||
'source',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'market_name',
|
||||
'is_indicative',
|
||||
'ts_ms',
|
||||
'slot',
|
||||
'market_slot',
|
||||
'mark_price',
|
||||
'oracle_price',
|
||||
'best_bid_price',
|
||||
'best_ask_price',
|
||||
'mid_price',
|
||||
'spread_quote',
|
||||
'spread_bps',
|
||||
'depth_levels',
|
||||
'bid_levels',
|
||||
'ask_levels',
|
||||
'top_bid_size',
|
||||
'top_ask_size',
|
||||
'top_bid_notional',
|
||||
'top_ask_notional',
|
||||
'depth_bid_base',
|
||||
'depth_ask_base',
|
||||
'depth_bid_quote',
|
||||
'depth_ask_quote',
|
||||
'imbalance',
|
||||
'bids_norm',
|
||||
'asks_norm',
|
||||
'raw_payload_hash',
|
||||
'inserted_at',
|
||||
]);
|
||||
|
||||
await ensurePublicSelectTable(dlobStatsTsTable, [
|
||||
'ts',
|
||||
'id',
|
||||
'source',
|
||||
'market_name',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'source_ts',
|
||||
'slot',
|
||||
'mark_price',
|
||||
'oracle_price',
|
||||
'best_bid_price',
|
||||
'best_ask_price',
|
||||
'mid_price',
|
||||
'spread_abs',
|
||||
'spread_bps',
|
||||
'depth_levels',
|
||||
'depth_bid_base',
|
||||
'depth_ask_base',
|
||||
'depth_bid_usd',
|
||||
'depth_ask_usd',
|
||||
'imbalance',
|
||||
'raw',
|
||||
], { publicFilter: dlobPublicFilter });
|
||||
|
||||
await ensurePublicSelectTable(dlobDepthBpsTsTable, [
|
||||
'ts',
|
||||
'id',
|
||||
'source',
|
||||
'market_name',
|
||||
'band_bps',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'source_ts',
|
||||
'slot',
|
||||
'mid_price',
|
||||
'best_bid_price',
|
||||
'best_ask_price',
|
||||
'bid_base',
|
||||
'ask_base',
|
||||
'bid_usd',
|
||||
'ask_usd',
|
||||
'imbalance',
|
||||
'raw',
|
||||
], { publicFilter: dlobPublicFilter });
|
||||
|
||||
await ensurePublicSelectTable(dlobSlippageTsTable, [
|
||||
'ts',
|
||||
'id',
|
||||
'source',
|
||||
'market_name',
|
||||
'side',
|
||||
'size_usd',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'source_ts',
|
||||
'slot',
|
||||
'mid_price',
|
||||
'vwap_price',
|
||||
'worst_price',
|
||||
'filled_usd',
|
||||
'filled_base',
|
||||
'impact_bps',
|
||||
'levels_consumed',
|
||||
'fill_pct',
|
||||
'raw',
|
||||
], { publicFilter: dlobPublicFilter });
|
||||
|
||||
await ensurePublicSelectTable(dlobSlippageTsV2Table, [
|
||||
'ts',
|
||||
'id',
|
||||
'source',
|
||||
'market_name',
|
||||
'side',
|
||||
'size_usd',
|
||||
'market_type',
|
||||
'market_index',
|
||||
'source_ts',
|
||||
'slot',
|
||||
'mid_price',
|
||||
'vwap_price',
|
||||
'worst_price',
|
||||
'filled_usd',
|
||||
'filled_base',
|
||||
'impact_bps',
|
||||
'levels_consumed',
|
||||
'fill_pct',
|
||||
'raw',
|
||||
], { publicFilter: dlobPublicFilter });
|
||||
|
||||
// Bot control-plane tables (tracked; permissions are intentionally not created here by default).
|
||||
for (const t of [botConfigTable, botStateTable, botEventsTable]) {
|
||||
await metadataIgnore({ type: 'pg_untrack_table', args: { source, table: t } });
|
||||
await metadata({ type: 'pg_track_table', args: { source, table: t } });
|
||||
}
|
||||
|
||||
// Return table type for candle functions (needed for Hasura to track the function).
|
||||
await metadataIgnore({ type: 'pg_track_table', args: { source, table: candlesReturnTable } });
|
||||
|
||||
try {
|
||||
await metadata({ type: 'pg_track_table', args: { source, table: apiTokensTable } });
|
||||
} catch (err) {
|
||||
@@ -175,7 +581,7 @@ async function main() {
|
||||
}
|
||||
|
||||
for (const fn of candleFns) {
|
||||
// Function for aggregated candle queries (used by trade-api).
|
||||
// Function for aggregated candle queries and indicator reads over Hasura.
|
||||
await metadataIgnore({ type: 'pg_untrack_function', args: { source, function: fn } });
|
||||
await metadata({ type: 'pg_track_function', args: { source, function: fn } });
|
||||
}
|
||||
|
||||
@@ -3,7 +3,9 @@ kind: Job
|
||||
metadata:
|
||||
name: hasura-bootstrap
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "2"
|
||||
argocd.argoproj.io/hook: Sync
|
||||
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded
|
||||
argocd.argoproj.io/sync-wave: "3"
|
||||
spec:
|
||||
backoffLimit: 5
|
||||
template:
|
||||
|
||||
@@ -1,49 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: trade-ingestor
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: trade-ingestor
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: trade-ingestor
|
||||
spec:
|
||||
imagePullSecrets:
|
||||
- name: gitea-registry
|
||||
containers:
|
||||
- name: ingestor
|
||||
image: rv32i.pl/trade/trade-ingestor:k3s-20260106013603
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: MARKET_NAME
|
||||
value: "PUMP-PERP"
|
||||
- name: INTERVAL_MS
|
||||
value: "1000"
|
||||
- name: SOURCE
|
||||
value: "drift_oracle"
|
||||
- name: INGEST_VIA
|
||||
value: "api"
|
||||
- name: INGEST_API_URL
|
||||
value: "http://trade-api:8787"
|
||||
volumeMounts:
|
||||
- name: tokens
|
||||
mountPath: /app/tokens
|
||||
readOnly: true
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
args:
|
||||
- >
|
||||
npm run ingest:oracle -- \
|
||||
--market-name "${MARKET_NAME}" \
|
||||
--interval-ms "${INTERVAL_MS}" \
|
||||
--source "${SOURCE}" \
|
||||
--ingest-via "${INGEST_VIA}"
|
||||
volumes:
|
||||
- name: tokens
|
||||
secret:
|
||||
secretName: trade-ingestor-tokens
|
||||
File diff suppressed because it is too large
Load Diff
@@ -5,14 +5,15 @@ resources:
|
||||
- cm-trade-deploy.yaml
|
||||
- postgres/service.yaml
|
||||
- postgres/statefulset.yaml
|
||||
- postgres/job-migrate.yaml
|
||||
- hasura/service.yaml
|
||||
- hasura/deployment.yaml
|
||||
- hasura/job-bootstrap.yaml
|
||||
- api/service.yaml
|
||||
- api/deployment.yaml
|
||||
- ingestor/deployment.yaml
|
||||
- frontend/service.yaml
|
||||
- frontend/deployment.yaml
|
||||
- dlob/redis.yaml
|
||||
- dlob/publisher-deployment.yaml
|
||||
- candles-cache-worker/deployment.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: postgres-initdb
|
||||
@@ -21,6 +22,9 @@ configMapGenerator:
|
||||
- name: hasura-bootstrap-script
|
||||
files:
|
||||
- hasura/hasura-bootstrap.mjs
|
||||
- name: candles-cache-worker-script
|
||||
files:
|
||||
- candles-cache-worker/worker.mjs
|
||||
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
|
||||
36
kustomize/base/postgres/job-migrate.yaml
Normal file
36
kustomize/base/postgres/job-migrate.yaml
Normal file
@@ -0,0 +1,36 @@
|
||||
apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: postgres-migrate
|
||||
annotations:
|
||||
argocd.argoproj.io/hook: Sync
|
||||
argocd.argoproj.io/hook-delete-policy: BeforeHookCreation,HookSucceeded
|
||||
argocd.argoproj.io/sync-wave: "2"
|
||||
spec:
|
||||
backoffLimit: 3
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: postgres-migrate
|
||||
image: postgres:16
|
||||
imagePullPolicy: IfNotPresent
|
||||
envFrom:
|
||||
- secretRef:
|
||||
name: trade-postgres
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
- |
|
||||
export PGPASSWORD="$POSTGRES_PASSWORD"
|
||||
until pg_isready -h postgres -U "$POSTGRES_USER" -d "$POSTGRES_DB"; do sleep 1; done
|
||||
psql -h postgres -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=1 -f /migrations/001_init.sql
|
||||
volumeMounts:
|
||||
- name: migrations
|
||||
mountPath: /migrations/001_init.sql
|
||||
subPath: 001_init.sql
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: migrations
|
||||
configMap:
|
||||
name: postgres-initdb
|
||||
@@ -0,0 +1,13 @@
|
||||
apiVersion: cert-manager.io/v1
|
||||
kind: Certificate
|
||||
metadata:
|
||||
name: monitoring-mpabi-pl
|
||||
namespace: monitoring
|
||||
spec:
|
||||
secretName: monitoring-mpabi-pl-tls
|
||||
issuerRef:
|
||||
kind: ClusterIssuer
|
||||
name: letsencrypt-prod
|
||||
dnsNames:
|
||||
- grafana.mpabi.pl
|
||||
- prometheus.mpabi.pl
|
||||
238
kustomize/infra/monitoring-extras/dashboard-agave-status.yaml
Normal file
238
kustomize/infra/monitoring-extras/dashboard-agave-status.yaml
Normal file
@@ -0,0 +1,238 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: grafana-dashboard-agave-status
|
||||
namespace: monitoring
|
||||
labels:
|
||||
grafana_dashboard: "1"
|
||||
data:
|
||||
agave-status.json: |-
|
||||
{
|
||||
"uid": "agave-status-mpabi",
|
||||
"title": "Agave @ mpabi",
|
||||
"timezone": "browser",
|
||||
"schemaVersion": 39,
|
||||
"version": 2,
|
||||
"time": { "from": "now-6h", "to": "now" },
|
||||
"timepicker": {
|
||||
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m"],
|
||||
"time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d"]
|
||||
},
|
||||
"refresh": "10s",
|
||||
"tags": ["agave", "solana", "mpabi"],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
"name": "instance",
|
||||
"type": "query",
|
||||
"label": "Node Exporter Instance",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"query": { "query": "label_values(up{job=\"mpabi-node-exporter\"}, instance)", "refId": "PromVarInstance" },
|
||||
"current": { "selected": false, "text": "10.66.66.1:9100", "value": "10.66.66.1:9100" }
|
||||
}
|
||||
]
|
||||
},
|
||||
"panels": [
|
||||
{
|
||||
"id": 1,
|
||||
"type": "stat",
|
||||
"title": "Geyser Metrics Target (Prometheus up)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "A", "expr": "up{job=\"mpabi-yellowstone-geyser\"}" }],
|
||||
"options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "horizontal", "textMode": "value" },
|
||||
"fieldConfig": {
|
||||
"defaults": { "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } },
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 0 }
|
||||
},
|
||||
{
|
||||
"id": 2,
|
||||
"type": "stat",
|
||||
"title": "agave-validator.service (systemd active)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "B", "expr": "node_systemd_unit_state{job=\"mpabi-node-exporter\",instance=\"$instance\",name=\"agave-validator.service\",state=\"active\"}" }],
|
||||
"options": { "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "orientation": "horizontal", "textMode": "value" },
|
||||
"fieldConfig": {
|
||||
"defaults": { "thresholds": { "mode": "absolute", "steps": [{ "color": "red", "value": null }, { "color": "green", "value": 1 }] } },
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 0 }
|
||||
},
|
||||
{
|
||||
"id": 3,
|
||||
"type": "timeseries",
|
||||
"title": "CPU Used (%)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "C", "expr": "100 - (avg by (instance) (rate(node_cpu_seconds_total{job=\"mpabi-node-exporter\",instance=\"$instance\",mode=\"idle\"}[5m])) * 100)" }],
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 6 }
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"type": "timeseries",
|
||||
"title": "Load (1m)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "D", "expr": "node_load1{job=\"mpabi-node-exporter\",instance=\"$instance\"}" }],
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 6 }
|
||||
},
|
||||
{
|
||||
"id": 5,
|
||||
"type": "timeseries",
|
||||
"title": "Memory Used (%)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "E", "expr": "100 - (node_memory_MemAvailable_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\"} / node_memory_MemTotal_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\"} * 100)" }],
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 6 }
|
||||
},
|
||||
{
|
||||
"id": 6,
|
||||
"type": "timeseries",
|
||||
"title": "Swap Used (GiB)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "F", "expr": "(node_memory_SwapTotal_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\"} - node_memory_SwapFree_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\"}) / 1024 / 1024 / 1024" }],
|
||||
"fieldConfig": { "defaults": { "unit": "gbytes" }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 0, "y": 14 }
|
||||
},
|
||||
{
|
||||
"id": 7,
|
||||
"type": "timeseries",
|
||||
"title": "Disk Free Accounts (%)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "G", "expr": "100 * node_filesystem_avail_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\",mountpoint=\"/var/lib/solana/accounts\"} / node_filesystem_size_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\",mountpoint=\"/var/lib/solana/accounts\"}" }],
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 8, "y": 14 }
|
||||
},
|
||||
{
|
||||
"id": 8,
|
||||
"type": "timeseries",
|
||||
"title": "Disk Free Ledger (%)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "H", "expr": "100 * node_filesystem_avail_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\",mountpoint=\"/var/lib/solana/ledger\"} / node_filesystem_size_bytes{job=\"mpabi-node-exporter\",instance=\"$instance\",mountpoint=\"/var/lib/solana/ledger\"}" }],
|
||||
"fieldConfig": { "defaults": { "unit": "percent", "min": 0, "max": 100 }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 8, "x": 16, "y": 14 }
|
||||
},
|
||||
{
|
||||
"id": 9,
|
||||
"type": "timeseries",
|
||||
"title": "Disk IO (NVMe) Read/Write (MiB/s)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "refId": "I", "expr": "sum by (device) (rate(node_disk_read_bytes_total{job=\"mpabi-node-exporter\",instance=\"$instance\",device=~\"nvme.*\"}[5m])) / 1024 / 1024", "legendFormat": "read {{device}}" },
|
||||
{ "refId": "J", "expr": "sum by (device) (rate(node_disk_written_bytes_total{job=\"mpabi-node-exporter\",instance=\"$instance\",device=~\"nvme.*\"}[5m])) / 1024 / 1024", "legendFormat": "write {{device}}" }
|
||||
],
|
||||
"fieldConfig": { "defaults": { "unit": "mbytes" }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 22 }
|
||||
},
|
||||
{
|
||||
"id": 10,
|
||||
"type": "timeseries",
|
||||
"title": "Network wg0 RX/TX (MiB/s)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "refId": "K", "expr": "rate(node_network_receive_bytes_total{job=\"mpabi-node-exporter\",instance=\"$instance\",device=\"wg0\"}[5m]) / 1024 / 1024", "legendFormat": "rx" },
|
||||
{ "refId": "L", "expr": "rate(node_network_transmit_bytes_total{job=\"mpabi-node-exporter\",instance=\"$instance\",device=\"wg0\"}[5m]) / 1024 / 1024", "legendFormat": "tx" }
|
||||
],
|
||||
"fieldConfig": { "defaults": { "unit": "mbytes" }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 22 }
|
||||
},
|
||||
{
|
||||
"id": 11,
|
||||
"type": "timeseries",
|
||||
"title": "Geyser: Subscriber Queue Size",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "M", "expr": "grpc_subscriber_queue_size{job=\"mpabi-yellowstone-geyser\"}", "legendFormat": "{{subscriber_id}}" }],
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 30 }
|
||||
},
|
||||
{
|
||||
"id": 12,
|
||||
"type": "timeseries",
|
||||
"title": "Geyser: Connections Total",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "N", "expr": "connections_total{job=\"mpabi-yellowstone-geyser\"}" }],
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 30 }
|
||||
},
|
||||
{
|
||||
"id": 13,
|
||||
"type": "timeseries",
|
||||
"title": "Geyser: Bytes Sent (MiB/s)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "O", "expr": "rate(grpc_bytes_sent{job=\"mpabi-yellowstone-geyser\"}[5m]) / 1024 / 1024", "legendFormat": "{{subscriber_id}}" }],
|
||||
"fieldConfig": { "defaults": { "unit": "mbytes" }, "overrides": [] },
|
||||
"gridPos": { "h": 8, "w": 12, "x": 0, "y": 38 }
|
||||
},
|
||||
{
|
||||
"id": 14,
|
||||
"type": "timeseries",
|
||||
"title": "Geyser: Messages Sent (/s)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "P", "expr": "rate(grpc_message_sent_count{job=\"mpabi-yellowstone-geyser\"}[5m])", "legendFormat": "{{subscriber_id}}" }],
|
||||
"gridPos": { "h": 8, "w": 12, "x": 12, "y": 38 }
|
||||
},
|
||||
{
|
||||
"id": 15,
|
||||
"type": "timeseries",
|
||||
"title": "Geyser: Disconnects (increase 15m)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "Q", "expr": "sum by (reason) (increase(grpc_client_disconnects_total{job=\"mpabi-yellowstone-geyser\"}[15m]))", "legendFormat": "{{reason}}" }],
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 46 }
|
||||
},
|
||||
{
|
||||
"id": 16,
|
||||
"type": "stat",
|
||||
"title": "RPC Slot Lag (slots)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "R", "expr": "solana_rpc_slot_lag{job=\"mpabi-node-exporter\",instance=\"$instance\"}" }],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "short",
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 20 },
|
||||
{ "color": "red", "value": 50 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 6, "w": 12, "x": 0, "y": 54 }
|
||||
},
|
||||
{
|
||||
"id": 17,
|
||||
"type": "stat",
|
||||
"title": "RPC Slot Lag (szac. minuty)",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [{ "refId": "S", "expr": "solana_rpc_slot_lag{job=\"mpabi-node-exporter\",instance=\"$instance\"} * 0.4 / 60" }],
|
||||
"fieldConfig": {
|
||||
"defaults": {
|
||||
"unit": "min",
|
||||
"decimals": 2,
|
||||
"thresholds": {
|
||||
"mode": "absolute",
|
||||
"steps": [
|
||||
{ "color": "green", "value": null },
|
||||
{ "color": "yellow", "value": 1 },
|
||||
{ "color": "red", "value": 2 }
|
||||
]
|
||||
}
|
||||
},
|
||||
"overrides": []
|
||||
},
|
||||
"gridPos": { "h": 6, "w": 12, "x": 12, "y": 54 }
|
||||
},
|
||||
{
|
||||
"id": 18,
|
||||
"type": "timeseries",
|
||||
"title": "RPC Slot & Reference Slot",
|
||||
"datasource": { "type": "prometheus", "uid": "prometheus" },
|
||||
"targets": [
|
||||
{ "refId": "T", "expr": "solana_rpc_slot{job=\"mpabi-node-exporter\",instance=\"$instance\"}" },
|
||||
{ "refId": "U", "expr": "solana_rpc_slot_reference{job=\"mpabi-node-exporter\",instance=\"$instance\"}" },
|
||||
{ "refId": "V", "expr": "solana_rpc_block_height{job=\"mpabi-node-exporter\",instance=\"$instance\"}" }
|
||||
],
|
||||
"gridPos": { "h": 8, "w": 24, "x": 0, "y": 60 }
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: grafana-http
|
||||
namespace: monitoring
|
||||
spec:
|
||||
entryPoints:
|
||||
- web
|
||||
routes:
|
||||
- match: Host(`grafana.mpabi.pl`)
|
||||
kind: Rule
|
||||
middlewares:
|
||||
- name: redirect-to-https
|
||||
services:
|
||||
- name: monitoring-stack-grafana
|
||||
port: 80
|
||||
16
kustomize/infra/monitoring-extras/ingressroute-grafana.yaml
Normal file
16
kustomize/infra/monitoring-extras/ingressroute-grafana.yaml
Normal file
@@ -0,0 +1,16 @@
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: grafana
|
||||
namespace: monitoring
|
||||
spec:
|
||||
entryPoints:
|
||||
- websecure
|
||||
routes:
|
||||
- match: Host(`grafana.mpabi.pl`)
|
||||
kind: Rule
|
||||
services:
|
||||
- name: monitoring-stack-grafana
|
||||
port: 80
|
||||
tls:
|
||||
secretName: monitoring-mpabi-pl-tls
|
||||
@@ -0,0 +1,16 @@
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: prometheus-http
|
||||
namespace: monitoring
|
||||
spec:
|
||||
entryPoints:
|
||||
- web
|
||||
routes:
|
||||
- match: Host(`prometheus.mpabi.pl`)
|
||||
kind: Rule
|
||||
middlewares:
|
||||
- name: redirect-to-https
|
||||
services:
|
||||
- name: monitoring-stack-kube-prom-prometheus
|
||||
port: 9090
|
||||
@@ -0,0 +1,16 @@
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: IngressRoute
|
||||
metadata:
|
||||
name: prometheus
|
||||
namespace: monitoring
|
||||
spec:
|
||||
entryPoints:
|
||||
- websecure
|
||||
routes:
|
||||
- match: Host(`prometheus.mpabi.pl`)
|
||||
kind: Rule
|
||||
services:
|
||||
- name: monitoring-stack-kube-prom-prometheus
|
||||
port: 9090
|
||||
tls:
|
||||
secretName: monitoring-mpabi-pl-tls
|
||||
11
kustomize/infra/monitoring-extras/kustomization.yaml
Normal file
11
kustomize/infra/monitoring-extras/kustomization.yaml
Normal file
@@ -0,0 +1,11 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
resources:
|
||||
- middleware-redirect-to-https.yaml
|
||||
- certificate-monitoring-mpabi-pl.yaml
|
||||
- ingressroute-grafana.yaml
|
||||
- ingressroute-grafana-http.yaml
|
||||
- ingressroute-prometheus.yaml
|
||||
- ingressroute-prometheus-http.yaml
|
||||
- dashboard-agave-status.yaml
|
||||
- prometheus-rules-agave.yaml
|
||||
@@ -0,0 +1,9 @@
|
||||
apiVersion: traefik.io/v1alpha1
|
||||
kind: Middleware
|
||||
metadata:
|
||||
name: redirect-to-https
|
||||
namespace: monitoring
|
||||
spec:
|
||||
redirectScheme:
|
||||
scheme: https
|
||||
permanent: true
|
||||
@@ -0,0 +1,61 @@
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: agave-rpc-alerts
|
||||
namespace: monitoring
|
||||
labels:
|
||||
app: kube-prometheus-stack
|
||||
release: monitoring-stack
|
||||
spec:
|
||||
groups:
|
||||
- name: agave-rpc-health
|
||||
rules:
|
||||
- alert: AgaveRPCDown
|
||||
expr: "max by (instance) (solana_rpc_up{job=\"mpabi-node-exporter\"}) == 0"
|
||||
for: 30s
|
||||
labels:
|
||||
severity: critical
|
||||
team: mpabi
|
||||
annotations:
|
||||
summary: "Agave RPC is unreachable"
|
||||
description: "RPC probe from node exporter reports solana_rpc_up == 0 for instance {{ $labels.instance }}."
|
||||
- alert: AgaveRPCSlotLagHigh
|
||||
expr: "sum by (instance) (solana_rpc_slot_lag{job=\"mpabi-node-exporter\"}) > 50"
|
||||
for: 2m
|
||||
labels:
|
||||
severity: warning
|
||||
team: mpabi
|
||||
annotations:
|
||||
summary: "Agave RPC is lagging behind cluster"
|
||||
description: "Current slot lag is {{ $value }} for instance {{ $labels.instance }}. Reference endpoint in probe config may be misconfigured or validator is behind."
|
||||
- alert: AgaveRPCSlotLagCritical
|
||||
expr: "sum by (instance) (solana_rpc_slot_lag{job=\"mpabi-node-exporter\"}) > 500"
|
||||
for: 2m
|
||||
labels:
|
||||
severity: critical
|
||||
team: mpabi
|
||||
annotations:
|
||||
summary: "Agave RPC severe lag"
|
||||
description: "Slot lag is critically high ({{ $value }} slots) on instance {{ $labels.instance }}."
|
||||
- alert: AgaveIOHigh
|
||||
expr: |
|
||||
sum by (instance) (
|
||||
(rate(node_disk_read_bytes_total{job="mpabi-node-exporter",device=~"nvme.*"}[5m]) +
|
||||
rate(node_disk_written_bytes_total{job="mpabi-node-exporter",device=~"nvme.*"}[5m])) / 1024 / 1024
|
||||
) > 300
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
team: mpabi
|
||||
annotations:
|
||||
summary: "High storage I/O on Agave node"
|
||||
description: "Combined NVMe read+write throughput >300 MiB/s for 5m on {{ $labels.instance }}. Check disk pressure and Geyser/ledger workload."
|
||||
- alert: AgaveIOWaitHigh
|
||||
expr: "avg by (instance) (rate(node_cpu_seconds_total{job=\"mpabi-node-exporter\",mode=\"iowait\"}[5m])) > 0.2"
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
team: mpabi
|
||||
annotations:
|
||||
summary: "High iowait on Agave node"
|
||||
description: "Iowait over 20% on average for 5m on {{ $labels.instance }}. Storage latency is likely impacting slot progress."
|
||||
@@ -91,10 +91,10 @@ spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- portainer.rv32i.pl
|
||||
secretName: portainer-rv32i-pl-tls
|
||||
- portainer.mpabi.pl
|
||||
secretName: portainer-mpabi-pl-tls
|
||||
rules:
|
||||
- host: portainer.rv32i.pl
|
||||
- host: portainer.mpabi.pl
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
|
||||
83
kustomize/overlays/mevnode-bot/README.md
Normal file
83
kustomize/overlays/mevnode-bot/README.md
Normal file
@@ -0,0 +1,83 @@
|
||||
# mevnode-bot overlay
|
||||
|
||||
To jest docelowy overlay dla runtime `mevnode_bot`.
|
||||
|
||||
Zakres:
|
||||
|
||||
- `postgres`
|
||||
- `hasura`
|
||||
- `trade-frontend` (`NodePort 30081`)
|
||||
- `dlob-redis`
|
||||
- `dlob-publisher-hot`
|
||||
- `dlob-publisher-all`
|
||||
- `dlob-hot-redis-to-postgres-raw-writer`
|
||||
- `dlob-hot-postgres-to-postgres-derived-writer`
|
||||
- `dlob-all-redis-to-postgres-derived-writer`
|
||||
|
||||
Założenia runtime:
|
||||
|
||||
- `mevnode_sol` wystawia po `wg0`:
|
||||
- `RPC`: `http://10.91.0.1:8899`
|
||||
- `WS`: `ws://10.91.0.1:8900`
|
||||
- `Yellowstone`: `http://10.91.0.1:10000`
|
||||
- `mevnode_bot` uruchamia tylko warstwę aplikacyjną.
|
||||
- `trade-frontend` jest wystawiany przez `NodePort 30081`.
|
||||
- Namespace pozostaje `trade-staging`.
|
||||
|
||||
Topologia DLOB w tym overlayu:
|
||||
|
||||
- `dlob-publisher-hot` to hot-path na wybrane markety (`PERP_MARKETS_TO_LOAD=0,20,72`, czyli `SOL-PERP`, `JTO-PERP`, `ADA-PERP`) i prefix Redis `DLOB_HOT` (`dlob-hot:`).
|
||||
- `dlob-publisher-all` to szeroki feed do wszystkich marketów na prefix Redis `DLOB_ALL` (`dlob-all:`).
|
||||
- `dlob-hot-redis-to-postgres-raw-writer` to schema-first writer: czyta `dlob-hot:` z Redis i zapisuje kanoniczny raw DLOB do PostgreSQL (`dlob_hot_snapshot_latest`, `dlob_hot_snapshot_ts`).
|
||||
- `dlob-hot-postgres-to-postgres-derived-writer` czyta raw hot z PostgreSQL i buduje normalized layer (`dlob_hot_derived_latest`, `dlob_hot_derived_ts`).
|
||||
- `LIVE` na `k3s`: sciezka `hot` jest juz aktywna jako `Redis -> raw PG -> derived PG`.
|
||||
- `dlob-publisher-all` celowo nie ma patcha persistent store, żeby nie dublowac zapisow do Postgresa.
|
||||
- `dlob-all-redis-to-postgres-derived-writer` zapisuje dla `all` tylko warstwe pochodna do PostgreSQL; bez `raw_ts`, zeby nie zjadac dysku w kilka dni.
|
||||
- `LIVE` na `k3s`: sciezka `all` jest aktywna jako `Redis -> derived PG`.
|
||||
- `all` zapisuje pelne `derived_ts` dla wszystkich marketow z `dlob-all:*`, bez samplingu i bez shortlisty marketow.
|
||||
|
||||
Planowany zakres zapisu dla `all`:
|
||||
|
||||
- tozsamosc i czas: `source`, `market_type`, `market_index`, `market_name`, `is_indicative`, `event_ts/ts_ms`, `slot`, `market_slot`
|
||||
- cena i mikrostruktura: `mark_price`, `oracle_price`, `best_bid_price`, `best_ask_price`, `mid_price`, `spread_quote`, `spread_bps`
|
||||
- plynnosc: `depth_levels`, `bid_levels`, `ask_levels`, `top_bid_size`, `top_ask_size`, `top_bid_notional`, `top_ask_notional`, `depth_bid_base`, `depth_ask_base`, `depth_bid_quote`, `depth_ask_quote`, `imbalance`
|
||||
- drabinka do modelu/UI: `bids_norm`, `asks_norm` jako top `10` leveli z polami `price`, `sizeBase`, `notional`, `sources`
|
||||
- kontrola jakosci i dedupe: `raw_payload_hash`, `updated_at` / `inserted_at`
|
||||
- celowo pomijamy: pelny `raw payload`, `raw latest`, `raw ts`, `L3`, `best_makers`
|
||||
|
||||
Polityka `hot` vs `all`:
|
||||
|
||||
- `hot` i `all` rozniamy po roli operacyjnej, nie po samym mechanizmie subskrypcji.
|
||||
- `hot` = `dlob-publisher-hot`: subset marketow, najnizsza latencja, najwyzszy priorytet operacyjny.
|
||||
- `all` = `dlob-publisher-all`: pelne pokrycie marketow, feed do coverage/read-side/analityki, nizszy priorytet operacyjny niz `hot`.
|
||||
- `USE_ORDER_SUBSCRIBER=true` moze byc wlaczone na obu publisherach; to nie definiuje roli `hot` vs `all`.
|
||||
- `hot` powinien byc broniony jako pierwszy przy problemach z CPU/RAM/RPC i nie powinien byc degradowany przed `all`.
|
||||
- Jesli trzeba ograniczac load albo robic fallback, najpierw degradujemy `all`, dopiero potem `hot`.
|
||||
|
||||
Zalecana polityka runtime:
|
||||
|
||||
- `hot`: utrzymywac `OrderSubscriber + gRPC` jako sciezke docelowa.
|
||||
- `all`: domyslnie tez `OrderSubscriber + gRPC`, ale to pierwszy kandydat do przejscia na slabszy tryb przy presji zasobow.
|
||||
- Kolejnosc degradacji dla `all`: `gRPC -> websocket -> polling`.
|
||||
- Wylaczenie `USE_ORDER_SUBSCRIBER` traktowac jako tryb awaryjny/testowy, nie jako stan docelowy.
|
||||
|
||||
Wymagane sekrety:
|
||||
|
||||
- `trade-postgres`
|
||||
- `trade-hasura`
|
||||
- `trade-frontend-tokens`
|
||||
- `trade-dlob-rpc`
|
||||
|
||||
Opcjonalne:
|
||||
|
||||
- `gitea-registry` nie jest tu wymagane, bo overlay czyści `imagePullSecrets`.
|
||||
|
||||
Uwagi:
|
||||
|
||||
- Overlay zastępuje lokalny `_tmp/bot-rx-stack`.
|
||||
- `trade-deploy/kustomize/overlays/mevnode-bot` jest teraz źródłem prawdy dla runtime `mevnode_bot`.
|
||||
- Ten overlay nadal korzysta z plików z `../../base` w `configMapGenerator`, więc przy lokalnym `kustomize` może być potrzebne:
|
||||
|
||||
```bash
|
||||
sudo k3s kubectl kustomize trade-deploy/kustomize/overlays/mevnode-bot --load-restrictor LoadRestrictionsNone | sudo k3s kubectl apply -f -
|
||||
```
|
||||
@@ -0,0 +1,66 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-all-redis-to-postgres-derived-writer
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "5"
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: dlob-all-redis-to-postgres-derived-writer
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: dlob-all-redis-to-postgres-derived-writer
|
||||
spec:
|
||||
priorityClassName: dlob-all
|
||||
imagePullSecrets: []
|
||||
containers:
|
||||
- name: writer
|
||||
image: gitea.mpabi.pl/trade/trade-dlob-server:all-derived-20260317-014016
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: DLOB_SOURCE
|
||||
value: mevnode_bot_all_derived
|
||||
- name: REDIS_HOST
|
||||
value: dlob-redis
|
||||
- name: REDIS_PORT
|
||||
value: "6379"
|
||||
- name: REDIS_KEY_PREFIX
|
||||
value: "dlob-all:"
|
||||
- name: DLOB_POLL_MS
|
||||
value: "1000"
|
||||
- name: NORMALIZED_DEPTH
|
||||
value: "10"
|
||||
- name: PRICE_PRECISION
|
||||
value: "1000000"
|
||||
- name: BASE_PRECISION
|
||||
value: "1000000000"
|
||||
- name: PGHOST
|
||||
value: postgres
|
||||
- name: PGPORT
|
||||
value: "5432"
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_USER
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: PGDATABASE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_DB
|
||||
command: ["node", "/lib/scripts/dlobAllRedisToPostgresDerivedWriter.js"]
|
||||
resources:
|
||||
requests:
|
||||
cpu: 250m
|
||||
memory: 512Mi
|
||||
limits:
|
||||
cpu: 1000m
|
||||
memory: 2Gi
|
||||
@@ -0,0 +1,62 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-hot-postgres-to-postgres-derived-writer
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "5"
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: dlob-hot-postgres-to-postgres-derived-writer
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: dlob-hot-postgres-to-postgres-derived-writer
|
||||
spec:
|
||||
priorityClassName: dlob-hot
|
||||
imagePullSecrets: []
|
||||
containers:
|
||||
- name: writer
|
||||
image: gitea.mpabi.pl/trade/trade-dlob-server:hot-pg-events-20260320-205517
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: DLOB_SOURCE
|
||||
value: mevnode_bot_hot_derived
|
||||
- name: RAW_SOURCE
|
||||
value: mevnode_bot_hot_raw
|
||||
- name: DLOB_POLL_MS
|
||||
value: "5000"
|
||||
- name: NORMALIZED_DEPTH
|
||||
value: "10"
|
||||
- name: PRICE_PRECISION
|
||||
value: "1000000"
|
||||
- name: BASE_PRECISION
|
||||
value: "1000000000"
|
||||
- name: PGHOST
|
||||
value: postgres
|
||||
- name: PGPORT
|
||||
value: "5432"
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_USER
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: PGDATABASE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_DB
|
||||
command: ["node", "/lib/scripts/dlobHotPostgresToPostgresDerivedWriter.js"]
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
@@ -0,0 +1,60 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-hot-redis-to-postgres-raw-writer
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "5"
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: dlob-hot-redis-to-postgres-raw-writer
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: dlob-hot-redis-to-postgres-raw-writer
|
||||
spec:
|
||||
priorityClassName: dlob-hot
|
||||
imagePullSecrets: []
|
||||
containers:
|
||||
- name: writer
|
||||
image: gitea.mpabi.pl/trade/trade-dlob-server:hot-pg-events-20260320-205517
|
||||
imagePullPolicy: IfNotPresent
|
||||
env:
|
||||
- name: DLOB_SOURCE
|
||||
value: mevnode_bot_hot_raw
|
||||
- name: REDIS_HOST
|
||||
value: dlob-redis
|
||||
- name: REDIS_PORT
|
||||
value: "6379"
|
||||
- name: REDIS_KEY_PREFIX
|
||||
value: "dlob-hot:"
|
||||
- name: DLOB_POLL_MS
|
||||
value: "5000"
|
||||
- name: PGHOST
|
||||
value: postgres
|
||||
- name: PGPORT
|
||||
value: "5432"
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_USER
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: PGDATABASE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_DB
|
||||
command: ["node", "/lib/scripts/dlobHotRedisToPostgresRawWriter.js"]
|
||||
resources:
|
||||
requests:
|
||||
cpu: 100m
|
||||
memory: 256Mi
|
||||
limits:
|
||||
cpu: 500m
|
||||
memory: 1Gi
|
||||
@@ -0,0 +1,19 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: dlob-publisher-all
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "4"
|
||||
prometheus.io/scrape: "true"
|
||||
prometheus.io/port: "9464"
|
||||
prometheus.io/path: /metrics
|
||||
spec:
|
||||
selector:
|
||||
app.kubernetes.io/name: dlob-publisher-all
|
||||
ports:
|
||||
- name: http
|
||||
port: 8080
|
||||
targetPort: 8080
|
||||
- name: metrics
|
||||
port: 9464
|
||||
targetPort: 9464
|
||||
102
kustomize/overlays/mevnode-bot/dlob-publisher-all.yaml
Normal file
102
kustomize/overlays/mevnode-bot/dlob-publisher-all.yaml
Normal file
@@ -0,0 +1,102 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-publisher-all
|
||||
annotations:
|
||||
argocd.argoproj.io/sync-wave: "4"
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
app.kubernetes.io/name: dlob-publisher-all
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: dlob-publisher-all
|
||||
spec:
|
||||
imagePullSecrets: []
|
||||
containers:
|
||||
- name: publisher
|
||||
image: gitea.mpabi.pl/trade/trade-dlob-server:grpc-reconnect-20260319-023351
|
||||
imagePullPolicy: IfNotPresent
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 8080
|
||||
env:
|
||||
- name: RUNNING_LOCAL
|
||||
value: "true"
|
||||
- name: LOCAL_CACHE
|
||||
value: "true"
|
||||
- name: ENV
|
||||
value: mainnet-beta
|
||||
- name: USE_WEBSOCKET
|
||||
value: "true"
|
||||
- name: USE_ORDER_SUBSCRIBER
|
||||
value: "true"
|
||||
- name: USE_GRPC
|
||||
value: "true"
|
||||
- name: DISABLE_GPA_REFRESH
|
||||
value: "true"
|
||||
- name: KILLSWITCH_SLOT_DIFF_THRESHOLD
|
||||
value: "1000000000"
|
||||
- name: ENABLE_TOB_MONITORING
|
||||
value: "false"
|
||||
- name: ELASTICACHE_HOST
|
||||
value: dlob-redis
|
||||
- name: ELASTICACHE_PORT
|
||||
value: "6379"
|
||||
- name: REDIS_CLIENT
|
||||
value: DLOB_ALL
|
||||
- name: FETCH_CONNECT_TIMEOUT_MS
|
||||
value: "15000"
|
||||
- name: FETCH_HEADERS_TIMEOUT_MS
|
||||
value: "300000"
|
||||
- name: FETCH_BODY_TIMEOUT_MS
|
||||
value: "300000"
|
||||
- name: GRPC_CLIENT
|
||||
value: "yellowstone"
|
||||
- name: ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-dlob-rpc
|
||||
key: ENDPOINT
|
||||
- name: WS_ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-dlob-rpc
|
||||
key: WS_ENDPOINT
|
||||
- name: GRPC_ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-dlob-rpc
|
||||
key: GRPC_ENDPOINT
|
||||
- name: TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-dlob-rpc
|
||||
key: TOKEN
|
||||
command: ["node", "/lib/publishers/dlobPublisher.js"]
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /startup
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 180
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /startup
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 24
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 10
|
||||
63
kustomize/overlays/mevnode-bot/dlob-redis-retention.yaml
Normal file
63
kustomize/overlays/mevnode-bot/dlob-redis-retention.yaml
Normal file
@@ -0,0 +1,63 @@
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: dlob-redis-retention
|
||||
data:
|
||||
clean.sh: |
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
now_s="$(date +%s)"
|
||||
|
||||
for pattern in 'dlob-hot:*' 'dlob-all:*'; do
|
||||
max_age_s=900
|
||||
if [ "$pattern" = 'dlob-all:*' ]; then
|
||||
max_age_s=3600
|
||||
fi
|
||||
|
||||
for key in $(redis-cli -h dlob-redis --scan --pattern "$pattern"); do
|
||||
value="$(redis-cli -h dlob-redis --raw get "$key" || true)"
|
||||
ts="$(printf '%s' "$value" | sed -n 's/.*"ts":\([0-9][0-9]*\).*/\1/p')"
|
||||
if [ -z "$ts" ]; then
|
||||
ts="$(printf '%s' "$value" | sed -n 's/.*"updatedAtTs":\([0-9][0-9]*\).*/\1/p')"
|
||||
fi
|
||||
[ -n "$ts" ] || continue
|
||||
if [ $((now_s - ts / 1000)) -gt "$max_age_s" ]; then
|
||||
redis-cli -h dlob-redis del "$key" >/dev/null
|
||||
fi
|
||||
done
|
||||
done
|
||||
---
|
||||
apiVersion: batch/v1
|
||||
kind: CronJob
|
||||
metadata:
|
||||
name: dlob-redis-retention
|
||||
spec:
|
||||
schedule: "*/5 * * * *"
|
||||
successfulJobsHistoryLimit: 1
|
||||
failedJobsHistoryLimit: 1
|
||||
jobTemplate:
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
restartPolicy: OnFailure
|
||||
containers:
|
||||
- name: cleaner
|
||||
image: redis:7-alpine
|
||||
command:
|
||||
- /bin/sh
|
||||
- /scripts/clean.sh
|
||||
resources:
|
||||
requests:
|
||||
cpu: 10m
|
||||
memory: 32Mi
|
||||
limits:
|
||||
cpu: 100m
|
||||
memory: 128Mi
|
||||
volumeMounts:
|
||||
- name: script
|
||||
mountPath: /scripts
|
||||
volumes:
|
||||
- name: script
|
||||
configMap:
|
||||
name: dlob-redis-retention
|
||||
defaultMode: 365
|
||||
44
kustomize/overlays/mevnode-bot/kustomization.yaml
Normal file
44
kustomize/overlays/mevnode-bot/kustomization.yaml
Normal file
@@ -0,0 +1,44 @@
|
||||
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||
kind: Kustomization
|
||||
|
||||
namespace: trade-staging
|
||||
|
||||
resources:
|
||||
- namespace.yaml
|
||||
- ../../base/cm-trade-deploy.yaml
|
||||
- ../../base/postgres/service.yaml
|
||||
- ../../base/postgres/statefulset.yaml
|
||||
- ../../base/postgres/job-migrate.yaml
|
||||
- ../../base/hasura/service.yaml
|
||||
- ../../base/hasura/deployment.yaml
|
||||
- ../../base/hasura/job-bootstrap.yaml
|
||||
- ../../base/frontend/service.yaml
|
||||
- ../../base/frontend/deployment.yaml
|
||||
- ../../base/dlob/redis.yaml
|
||||
- priority-classes.yaml
|
||||
- ../../base/dlob/publisher-deployment.yaml
|
||||
- dlob-publisher-all.yaml
|
||||
- dlob-publisher-all-service.yaml
|
||||
- dlob-hot-redis-to-postgres-raw-writer.yaml
|
||||
- dlob-hot-postgres-to-postgres-derived-writer.yaml
|
||||
- dlob-all-redis-to-postgres-derived-writer.yaml
|
||||
- dlob-redis-retention.yaml
|
||||
|
||||
patchesStrategicMerge:
|
||||
- patch-frontend.yaml
|
||||
- patch-frontend-service.yaml
|
||||
- patch-dlob-publisher.yaml
|
||||
- patch-dlob-publisher-storage.yaml
|
||||
- patch-dlob-publisher-hot-priority.yaml
|
||||
- patch-dlob-publisher-all-priority.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: postgres-initdb
|
||||
files:
|
||||
- ../../base/initdb/001_init.sql
|
||||
- name: hasura-bootstrap-script
|
||||
files:
|
||||
- ../../base/hasura/hasura-bootstrap.mjs
|
||||
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
4
kustomize/overlays/mevnode-bot/namespace.yaml
Normal file
4
kustomize/overlays/mevnode-bot/namespace.yaml
Normal file
@@ -0,0 +1,4 @@
|
||||
apiVersion: v1
|
||||
kind: Namespace
|
||||
metadata:
|
||||
name: trade-staging
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-publisher-all
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
priorityClassName: dlob-all
|
||||
@@ -0,0 +1,8 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-publisher-hot
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
priorityClassName: dlob-hot
|
||||
@@ -0,0 +1,40 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-publisher-hot
|
||||
spec:
|
||||
replicas: 1
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: publisher
|
||||
env:
|
||||
- name: ENABLE_PERSISTENT_STORE
|
||||
value: "true"
|
||||
- name: DLOB_SOURCE
|
||||
value: "mevnode_bot_hot"
|
||||
- name: PRICE_PRECISION
|
||||
value: "1000000"
|
||||
- name: BASE_PRECISION
|
||||
value: "1000000000"
|
||||
- name: PERSISTENT_STATS_DEPTH
|
||||
value: "10"
|
||||
- name: PGHOST
|
||||
value: postgres
|
||||
- name: PGPORT
|
||||
value: "5432"
|
||||
- name: PGUSER
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_USER
|
||||
- name: PGPASSWORD
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_PASSWORD
|
||||
- name: PGDATABASE
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-postgres
|
||||
key: POSTGRES_DB
|
||||
61
kustomize/overlays/mevnode-bot/patch-dlob-publisher.yaml
Normal file
61
kustomize/overlays/mevnode-bot/patch-dlob-publisher.yaml
Normal file
@@ -0,0 +1,61 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: dlob-publisher-hot
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
imagePullSecrets: []
|
||||
containers:
|
||||
- name: publisher
|
||||
env:
|
||||
- name: PERP_MARKETS_TO_LOAD
|
||||
value: "0,20,72"
|
||||
- name: USE_GRPC
|
||||
value: "true"
|
||||
- name: USE_WEBSOCKET
|
||||
value: "true"
|
||||
- name: DISABLE_GPA_REFRESH
|
||||
value: "true"
|
||||
- name: FETCH_CONNECT_TIMEOUT_MS
|
||||
value: "15000"
|
||||
- name: FETCH_HEADERS_TIMEOUT_MS
|
||||
value: "300000"
|
||||
- name: FETCH_BODY_TIMEOUT_MS
|
||||
value: "300000"
|
||||
- name: GRPC_CLIENT
|
||||
value: "yellowstone"
|
||||
- name: GRPC_ENDPOINT
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-dlob-rpc
|
||||
key: GRPC_ENDPOINT
|
||||
- name: TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: trade-dlob-rpc
|
||||
key: TOKEN
|
||||
startupProbe:
|
||||
httpGet:
|
||||
path: /startup
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 90
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /startup
|
||||
port: http
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 10
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 12
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health
|
||||
port: http
|
||||
initialDelaySeconds: 30
|
||||
periodSeconds: 20
|
||||
timeoutSeconds: 3
|
||||
failureThreshold: 10
|
||||
11
kustomize/overlays/mevnode-bot/patch-frontend-service.yaml
Normal file
11
kustomize/overlays/mevnode-bot/patch-frontend-service.yaml
Normal file
@@ -0,0 +1,11 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: trade-frontend
|
||||
spec:
|
||||
type: NodePort
|
||||
ports:
|
||||
- name: http
|
||||
port: 8081
|
||||
targetPort: http
|
||||
nodePort: 30081
|
||||
8
kustomize/overlays/mevnode-bot/patch-frontend.yaml
Normal file
8
kustomize/overlays/mevnode-bot/patch-frontend.yaml
Normal file
@@ -0,0 +1,8 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: trade-frontend
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
imagePullSecrets: []
|
||||
15
kustomize/overlays/mevnode-bot/priority-classes.yaml
Normal file
15
kustomize/overlays/mevnode-bot/priority-classes.yaml
Normal file
@@ -0,0 +1,15 @@
|
||||
apiVersion: scheduling.k8s.io/v1
|
||||
kind: PriorityClass
|
||||
metadata:
|
||||
name: dlob-hot
|
||||
value: 100000
|
||||
globalDefault: false
|
||||
description: Higher scheduling priority for hot DLOB publisher.
|
||||
---
|
||||
apiVersion: scheduling.k8s.io/v1
|
||||
kind: PriorityClass
|
||||
metadata:
|
||||
name: dlob-all
|
||||
value: 50000
|
||||
globalDefault: false
|
||||
description: Lower scheduling priority for all-market DLOB publisher.
|
||||
18
kustomize/overlays/prod/frontend-graphql-proxy-patch.yaml
Normal file
18
kustomize/overlays/prod/frontend-graphql-proxy-patch.yaml
Normal file
@@ -0,0 +1,18 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: trade-frontend
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: frontend
|
||||
volumeMounts:
|
||||
- name: frontend-server-script
|
||||
mountPath: /app/services/frontend/server.mjs
|
||||
subPath: frontend-server.mjs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: frontend-server-script
|
||||
configMap:
|
||||
name: trade-frontend-server-script
|
||||
732
kustomize/overlays/prod/frontend-server.mjs
Normal file
732
kustomize/overlays/prod/frontend-server.mjs
Normal file
@@ -0,0 +1,732 @@
|
||||
import crypto from 'node:crypto';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import net from 'node:net';
|
||||
import path from 'node:path';
|
||||
import tls from 'node:tls';
|
||||
|
||||
const PORT = Number.parseInt(process.env.PORT || '8081', 10);
|
||||
if (!Number.isInteger(PORT) || PORT <= 0) throw new Error(`Invalid PORT: ${process.env.PORT}`);
|
||||
|
||||
const APP_VERSION = String(process.env.APP_VERSION || 'v1').trim() || 'v1';
|
||||
const BUILD_TIMESTAMP = String(process.env.BUILD_TIMESTAMP || '').trim() || undefined;
|
||||
const STARTED_AT = new Date().toISOString();
|
||||
|
||||
const STATIC_DIR = process.env.STATIC_DIR || '/srv';
|
||||
const BASIC_AUTH_FILE = process.env.BASIC_AUTH_FILE || '/tokens/frontend.json';
|
||||
const HASURA_UPSTREAM = process.env.HASURA_UPSTREAM || 'http://hasura:8080';
|
||||
const HASURA_GRAPHQL_PATH = process.env.HASURA_GRAPHQL_PATH || '/v1/graphql';
|
||||
const GRAPHQL_CORS_ORIGIN = process.env.GRAPHQL_CORS_ORIGIN || process.env.CORS_ORIGIN || '*';
|
||||
const BASIC_AUTH_MODE = String(process.env.BASIC_AUTH_MODE || 'on')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const BASIC_AUTH_ENABLED = !['off', 'false', '0', 'disabled', 'none'].includes(BASIC_AUTH_MODE);
|
||||
const AUTH_USER_HEADER = String(process.env.AUTH_USER_HEADER || 'x-trade-user')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const AUTH_MODE = String(process.env.AUTH_MODE || 'session')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const HTPASSWD_FILE = String(process.env.HTPASSWD_FILE || '/auth/users').trim();
|
||||
const AUTH_SESSION_SECRET_FILE = String(process.env.AUTH_SESSION_SECRET_FILE || '').trim() || null;
|
||||
const AUTH_SESSION_COOKIE = String(process.env.AUTH_SESSION_COOKIE || 'trade_session')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const AUTH_SESSION_TTL_SECONDS = Number.parseInt(process.env.AUTH_SESSION_TTL_SECONDS || '43200', 10); // 12h
|
||||
const DLOB_SOURCE_COOKIE = String(process.env.DLOB_SOURCE_COOKIE || 'trade_dlob_source').trim() || 'trade_dlob_source';
|
||||
const DLOB_SOURCE_DEFAULT = String(process.env.DLOB_SOURCE_DEFAULT || 'mevnode').trim() || 'mevnode';
|
||||
|
||||
function readJson(filePath) {
|
||||
const raw = fs.readFileSync(filePath, 'utf8');
|
||||
return JSON.parse(raw);
|
||||
}
|
||||
|
||||
function readText(filePath) {
|
||||
return fs.readFileSync(filePath, 'utf8');
|
||||
}
|
||||
|
||||
function timingSafeEqualStr(a, b) {
|
||||
const aa = Buffer.from(String(a), 'utf8');
|
||||
const bb = Buffer.from(String(b), 'utf8');
|
||||
if (aa.length !== bb.length) return false;
|
||||
return crypto.timingSafeEqual(aa, bb);
|
||||
}
|
||||
|
||||
function timingSafeEqualBuf(a, b) {
|
||||
if (!(a instanceof Uint8Array) || !(b instanceof Uint8Array)) return false;
|
||||
if (a.length !== b.length) return false;
|
||||
return crypto.timingSafeEqual(Buffer.from(a), Buffer.from(b));
|
||||
}
|
||||
|
||||
function loadBasicAuth() {
|
||||
const j = readJson(BASIC_AUTH_FILE);
|
||||
const username = (j?.username || '').toString();
|
||||
const password = (j?.password || '').toString();
|
||||
if (!username || !password) throw new Error(`Invalid BASIC_AUTH_FILE: ${BASIC_AUTH_FILE}`);
|
||||
return { username, password };
|
||||
}
|
||||
|
||||
function send(res, status, headers, body) {
|
||||
res.statusCode = status;
|
||||
for (const [k, v] of Object.entries(headers || {})) res.setHeader(k, v);
|
||||
if (body == null) return void res.end();
|
||||
res.end(body);
|
||||
}
|
||||
|
||||
function sendJson(res, status, body) {
|
||||
send(res, status, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' }, JSON.stringify(body));
|
||||
}
|
||||
|
||||
function basicAuthRequired(res) {
|
||||
res.setHeader('www-authenticate', 'Basic realm="trade"');
|
||||
send(res, 401, { 'content-type': 'text/plain; charset=utf-8' }, 'unauthorized');
|
||||
}
|
||||
|
||||
function unauthorized(res) {
|
||||
sendJson(res, 401, { ok: false, error: 'unauthorized' });
|
||||
}
|
||||
|
||||
function isAuthorized(req, creds) {
|
||||
const auth = req.headers.authorization || '';
|
||||
const m = String(auth).match(/^Basic\s+(.+)$/i);
|
||||
if (!m?.[1]) return false;
|
||||
let decoded;
|
||||
try {
|
||||
decoded = Buffer.from(m[1], 'base64').toString('utf8');
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
const idx = decoded.indexOf(':');
|
||||
if (idx < 0) return false;
|
||||
const u = decoded.slice(0, idx);
|
||||
const p = decoded.slice(idx + 1);
|
||||
return timingSafeEqualStr(u, creds.username) && timingSafeEqualStr(p, creds.password);
|
||||
}
|
||||
|
||||
const MIME = {
|
||||
'.html': 'text/html; charset=utf-8',
|
||||
'.css': 'text/css; charset=utf-8',
|
||||
'.js': 'application/javascript; charset=utf-8',
|
||||
'.mjs': 'application/javascript; charset=utf-8',
|
||||
'.json': 'application/json; charset=utf-8',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.png': 'image/png',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.gif': 'image/gif',
|
||||
'.ico': 'image/x-icon',
|
||||
'.txt': 'text/plain; charset=utf-8',
|
||||
'.map': 'application/json; charset=utf-8',
|
||||
};
|
||||
|
||||
function contentTypeFor(filePath) {
|
||||
return MIME[path.extname(filePath).toLowerCase()] || 'application/octet-stream';
|
||||
}
|
||||
|
||||
function safePathFromUrlPath(urlPath) {
|
||||
const decoded = decodeURIComponent(urlPath);
|
||||
const cleaned = decoded.replace(/\0/g, '');
|
||||
// strip leading slash so join() doesn't ignore STATIC_DIR
|
||||
const rel = cleaned.replace(/^\/+/, '');
|
||||
const normalized = path.normalize(rel);
|
||||
// prevent traversal
|
||||
if (normalized.startsWith('..') || path.isAbsolute(normalized)) return null;
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function injectIndexHtml(html, { dlobSource, redirectPath }) {
|
||||
const src = normalizeDlobSource(dlobSource) || 'mevnode';
|
||||
const redirect = safeRedirectPath(redirectPath);
|
||||
const hrefBase = `/prefs/dlob-source?redirect=${encodeURIComponent(redirect)}&set=`;
|
||||
|
||||
const styleActive = 'font-weight:700;text-decoration:underline;';
|
||||
const styleInactive = 'font-weight:400;text-decoration:none;';
|
||||
|
||||
const snippet = `
|
||||
<!-- trade: dlob source switch -->
|
||||
<div style="position:fixed;right:12px;bottom:12px;z-index:2147483647;background:rgba(0,0,0,0.72);color:#fff;padding:8px 10px;border-radius:10px;font:12px/1.2 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;backdrop-filter:blur(6px);">
|
||||
<span style="opacity:0.85;margin-right:6px;">DLOB</span>
|
||||
<a href="${hrefBase}mevnode" style="color:#fff;${src === 'mevnode' ? styleActive : styleInactive}">mevnode</a>
|
||||
<span style="opacity:0.6;margin:0 6px;">|</span>
|
||||
<a href="${hrefBase}drift" style="color:#fff;${src === 'drift' ? styleActive : styleInactive}">drift</a>
|
||||
</div>
|
||||
`;
|
||||
|
||||
const bodyClose = /<\/body>/i;
|
||||
if (bodyClose.test(html)) return html.replace(bodyClose, `${snippet}</body>`);
|
||||
return `${html}\n${snippet}\n`;
|
||||
}
|
||||
|
||||
function serveStatic(req, res) {
|
||||
if (req.method !== 'GET' && req.method !== 'HEAD') {
|
||||
send(res, 405, { 'content-type': 'text/plain; charset=utf-8' }, 'method_not_allowed');
|
||||
return;
|
||||
}
|
||||
|
||||
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||
const rel = safePathFromUrlPath(url.pathname);
|
||||
if (rel == null) {
|
||||
send(res, 400, { 'content-type': 'text/plain; charset=utf-8' }, 'bad_request');
|
||||
return;
|
||||
}
|
||||
|
||||
const root = path.resolve(STATIC_DIR);
|
||||
const fileCandidate = path.resolve(root, rel);
|
||||
if (!fileCandidate.startsWith(root)) {
|
||||
send(res, 400, { 'content-type': 'text/plain; charset=utf-8' }, 'bad_request');
|
||||
return;
|
||||
}
|
||||
|
||||
const trySend = (filePath) => {
|
||||
try {
|
||||
const st = fs.statSync(filePath);
|
||||
if (st.isDirectory()) return trySend(path.join(filePath, 'index.html'));
|
||||
res.statusCode = 200;
|
||||
res.setHeader('content-type', contentTypeFor(filePath));
|
||||
res.setHeader('cache-control', filePath.endsWith('index.html') ? 'no-cache' : 'public, max-age=31536000');
|
||||
if (req.method === 'HEAD') return void res.end();
|
||||
if (filePath.endsWith('index.html')) {
|
||||
const html = fs.readFileSync(filePath, 'utf8');
|
||||
const injected = injectIndexHtml(html, {
|
||||
dlobSource: resolveDlobSource(req),
|
||||
redirectPath: url.pathname + url.search,
|
||||
});
|
||||
res.end(injected);
|
||||
return true;
|
||||
}
|
||||
fs.createReadStream(filePath).pipe(res);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// exact file, otherwise SPA fallback
|
||||
if (trySend(fileCandidate)) return;
|
||||
const indexPath = path.join(root, 'index.html');
|
||||
if (trySend(indexPath)) return;
|
||||
|
||||
send(res, 404, { 'content-type': 'text/plain; charset=utf-8' }, 'not_found');
|
||||
}
|
||||
|
||||
function stripHopByHopHeaders(headers) {
|
||||
const hop = new Set([
|
||||
'connection',
|
||||
'keep-alive',
|
||||
'proxy-authenticate',
|
||||
'proxy-authorization',
|
||||
'te',
|
||||
'trailer',
|
||||
'transfer-encoding',
|
||||
'upgrade',
|
||||
]);
|
||||
const out = {};
|
||||
for (const [k, v] of Object.entries(headers || {})) {
|
||||
if (hop.has(k.toLowerCase())) continue;
|
||||
out[k] = v;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function readHeader(req, name) {
|
||||
const v = req.headers[String(name).toLowerCase()];
|
||||
return Array.isArray(v) ? v[0] : v;
|
||||
}
|
||||
|
||||
function readCookie(req, name) {
|
||||
const raw = typeof req.headers.cookie === 'string' ? req.headers.cookie : '';
|
||||
if (!raw) return null;
|
||||
const needle = `${name}=`;
|
||||
for (const part of raw.split(';')) {
|
||||
const t = part.trim();
|
||||
if (!t.startsWith(needle)) continue;
|
||||
return t.slice(needle.length) || null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function normalizeDlobSource(value) {
|
||||
const v = String(value ?? '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (v === 'mevnode') return 'mevnode';
|
||||
if (v === 'drift') return 'drift';
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolveDlobSource(req) {
|
||||
const fromCookie = normalizeDlobSource(readCookie(req, DLOB_SOURCE_COOKIE));
|
||||
if (fromCookie) return fromCookie;
|
||||
return normalizeDlobSource(DLOB_SOURCE_DEFAULT) || 'mevnode';
|
||||
}
|
||||
|
||||
function safeRedirectPath(value) {
|
||||
const s = String(value ?? '').trim();
|
||||
if (!s.startsWith('/')) return '/';
|
||||
if (s.startsWith('//')) return '/';
|
||||
return s.replace(/\r|\n/g, '');
|
||||
}
|
||||
|
||||
function setDlobSourceCookie(res, { secure, dlobSource }) {
|
||||
const src = normalizeDlobSource(dlobSource);
|
||||
if (!src) return false;
|
||||
const parts = [
|
||||
`${DLOB_SOURCE_COOKIE}=${src}`,
|
||||
'Path=/',
|
||||
'SameSite=Lax',
|
||||
'HttpOnly',
|
||||
'Max-Age=31536000',
|
||||
];
|
||||
if (secure) parts.push('Secure');
|
||||
res.setHeader('set-cookie', parts.join('; '));
|
||||
return true;
|
||||
}
|
||||
|
||||
function resolveAuthUser(req) {
|
||||
const user = readHeader(req, AUTH_USER_HEADER) || readHeader(req, 'x-webauth-user');
|
||||
const value = typeof user === 'string' ? user.trim() : '';
|
||||
return value || null;
|
||||
}
|
||||
|
||||
function isHttpsRequest(req) {
|
||||
const xf = readHeader(req, 'x-forwarded-proto');
|
||||
if (typeof xf === 'string' && xf.toLowerCase() === 'https') return true;
|
||||
return Boolean(req.socket && req.socket.encrypted);
|
||||
}
|
||||
|
||||
function base64urlEncode(buf) {
|
||||
return Buffer.from(buf)
|
||||
.toString('base64')
|
||||
.replace(/\+/g, '-')
|
||||
.replace(/\//g, '_')
|
||||
.replace(/=+$/g, '');
|
||||
}
|
||||
|
||||
function base64urlDecode(str) {
|
||||
const cleaned = String(str).replace(/-/g, '+').replace(/_/g, '/');
|
||||
const pad = cleaned.length % 4 === 0 ? '' : '='.repeat(4 - (cleaned.length % 4));
|
||||
return Buffer.from(cleaned + pad, 'base64');
|
||||
}
|
||||
|
||||
function loadSessionSecret() {
|
||||
if (process.env.AUTH_SESSION_SECRET && String(process.env.AUTH_SESSION_SECRET).trim()) {
|
||||
return Buffer.from(String(process.env.AUTH_SESSION_SECRET).trim(), 'utf8');
|
||||
}
|
||||
if (AUTH_SESSION_SECRET_FILE) {
|
||||
try {
|
||||
const txt = readText(AUTH_SESSION_SECRET_FILE).trim();
|
||||
if (txt) return Buffer.from(txt, 'utf8');
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
return crypto.randomBytes(32);
|
||||
}
|
||||
|
||||
const SESSION_SECRET = loadSessionSecret();
|
||||
|
||||
function signSessionPayload(payloadB64) {
|
||||
return crypto.createHmac('sha256', SESSION_SECRET).update(payloadB64).digest();
|
||||
}
|
||||
|
||||
function makeSessionCookieValue(username) {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const exp = now + (Number.isFinite(AUTH_SESSION_TTL_SECONDS) && AUTH_SESSION_TTL_SECONDS > 0 ? AUTH_SESSION_TTL_SECONDS : 43200);
|
||||
const payload = JSON.stringify({ u: String(username), exp });
|
||||
const payloadB64 = base64urlEncode(Buffer.from(payload, 'utf8'));
|
||||
const sigB64 = base64urlEncode(signSessionPayload(payloadB64));
|
||||
return `${payloadB64}.${sigB64}`;
|
||||
}
|
||||
|
||||
function getSessionUser(req) {
|
||||
const raw = readCookie(req, AUTH_SESSION_COOKIE);
|
||||
if (!raw) return null;
|
||||
const parts = raw.split('.');
|
||||
if (parts.length !== 2) return null;
|
||||
const [payloadB64, sigB64] = parts;
|
||||
if (!payloadB64 || !sigB64) return null;
|
||||
|
||||
let payload;
|
||||
try {
|
||||
payload = JSON.parse(base64urlDecode(payloadB64).toString('utf8'));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const u = typeof payload?.u === 'string' ? payload.u.trim() : '';
|
||||
const exp = Number(payload?.exp);
|
||||
if (!u || !Number.isFinite(exp)) return null;
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
if (now >= exp) return null;
|
||||
|
||||
const expected = signSessionPayload(payloadB64);
|
||||
let got;
|
||||
try {
|
||||
got = base64urlDecode(sigB64);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!timingSafeEqualBuf(expected, got)) return null;
|
||||
|
||||
return u;
|
||||
}
|
||||
|
||||
function resolveAuthenticatedUser(req) {
|
||||
const sessionUser = getSessionUser(req);
|
||||
if (sessionUser) return sessionUser;
|
||||
const headerUser = resolveAuthUser(req);
|
||||
if (headerUser) return headerUser;
|
||||
if (AUTH_MODE === 'off' || AUTH_MODE === 'none' || AUTH_MODE === 'disabled') return 'anonymous';
|
||||
return null;
|
||||
}
|
||||
|
||||
function clearSessionCookie(res, secure) {
|
||||
const parts = [`${AUTH_SESSION_COOKIE}=`, 'Path=/', 'Max-Age=0', 'HttpOnly', 'SameSite=Lax'];
|
||||
if (secure) parts.push('Secure');
|
||||
res.setHeader('set-cookie', parts.join('; '));
|
||||
}
|
||||
|
||||
function setSessionCookie(res, secure, username) {
|
||||
const value = makeSessionCookieValue(username);
|
||||
const parts = [
|
||||
`${AUTH_SESSION_COOKIE}=${value}`,
|
||||
'Path=/',
|
||||
`Max-Age=${Number.isFinite(AUTH_SESSION_TTL_SECONDS) ? AUTH_SESSION_TTL_SECONDS : 43200}`,
|
||||
'HttpOnly',
|
||||
'SameSite=Lax',
|
||||
];
|
||||
if (secure) parts.push('Secure');
|
||||
res.setHeader('set-cookie', parts.join('; '));
|
||||
}
|
||||
|
||||
function verifyWithHtpasswd(username, password) {
|
||||
try {
|
||||
const r = spawnSync('htpasswd', ['-vb', HTPASSWD_FILE, String(username), String(password)], {
|
||||
stdio: 'ignore',
|
||||
timeout: 3000,
|
||||
});
|
||||
return r.status === 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readBody(req, limitBytes = 1024 * 16) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let total = 0;
|
||||
const chunks = [];
|
||||
req.on('data', (chunk) => {
|
||||
total += chunk.length;
|
||||
if (total > limitBytes) {
|
||||
reject(new Error('payload_too_large'));
|
||||
req.destroy();
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
});
|
||||
req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function withCors(res) {
|
||||
res.setHeader('access-control-allow-origin', GRAPHQL_CORS_ORIGIN);
|
||||
res.setHeader('access-control-allow-methods', 'GET,POST,OPTIONS');
|
||||
res.setHeader(
|
||||
'access-control-allow-headers',
|
||||
'content-type, authorization, x-hasura-admin-secret, x-hasura-role, x-hasura-user-id, x-hasura-dlob-source'
|
||||
);
|
||||
}
|
||||
|
||||
function proxyGraphqlHttp(req, res) {
|
||||
const upstreamBase = new URL(HASURA_UPSTREAM);
|
||||
const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||
|
||||
const target = new URL(upstreamBase.toString());
|
||||
target.pathname = HASURA_GRAPHQL_PATH;
|
||||
target.search = inUrl.search;
|
||||
|
||||
const isHttps = target.protocol === 'https:';
|
||||
const lib = isHttps ? https : http;
|
||||
|
||||
const headers = stripHopByHopHeaders(req.headers);
|
||||
headers.host = target.host;
|
||||
delete headers['x-hasura-dlob-source'];
|
||||
headers['x-hasura-dlob-source'] = resolveDlobSource(req);
|
||||
|
||||
const upstreamReq = lib.request(
|
||||
{
|
||||
protocol: target.protocol,
|
||||
hostname: target.hostname,
|
||||
port: target.port || (isHttps ? 443 : 80),
|
||||
method: req.method,
|
||||
path: target.pathname + target.search,
|
||||
headers,
|
||||
},
|
||||
(upstreamRes) => {
|
||||
const outHeaders = stripHopByHopHeaders(upstreamRes.headers);
|
||||
withCors(res);
|
||||
res.writeHead(upstreamRes.statusCode || 502, outHeaders);
|
||||
upstreamRes.pipe(res);
|
||||
}
|
||||
);
|
||||
|
||||
upstreamReq.on('error', (err) => {
|
||||
if (!res.headersSent) {
|
||||
withCors(res);
|
||||
send(res, 502, { 'content-type': 'text/plain; charset=utf-8' }, `bad_gateway: ${err?.message || err}`);
|
||||
} else {
|
||||
res.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
req.pipe(upstreamReq);
|
||||
}
|
||||
|
||||
function isGraphqlPath(pathname) {
|
||||
return pathname === '/graphql' || pathname === '/graphql-ws';
|
||||
}
|
||||
|
||||
function proxyGraphqlWs(req, socket, head) {
|
||||
const upstreamBase = new URL(HASURA_UPSTREAM);
|
||||
const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||
|
||||
const target = new URL(upstreamBase.toString());
|
||||
target.pathname = HASURA_GRAPHQL_PATH;
|
||||
target.search = inUrl.search;
|
||||
|
||||
const port = Number(target.port || (target.protocol === 'https:' ? 443 : 80));
|
||||
const host = target.hostname;
|
||||
|
||||
const connect =
|
||||
target.protocol === 'https:'
|
||||
? () => tls.connect({ host, port, servername: host })
|
||||
: () => net.connect({ host, port });
|
||||
|
||||
const upstream = connect();
|
||||
upstream.setNoDelay(true);
|
||||
socket.setNoDelay(true);
|
||||
|
||||
// For WebSocket upgrades we must forward `connection`/`upgrade` and related headers.
|
||||
const headers = { ...req.headers };
|
||||
delete headers['content-length'];
|
||||
delete headers['content-type'];
|
||||
headers.host = target.host;
|
||||
delete headers['x-hasura-dlob-source'];
|
||||
headers['x-hasura-dlob-source'] = resolveDlobSource(req);
|
||||
|
||||
const lines = [];
|
||||
lines.push(`GET ${target.pathname + target.search} HTTP/1.1`);
|
||||
for (const [k, v] of Object.entries(headers)) {
|
||||
if (v == null) continue;
|
||||
if (Array.isArray(v)) {
|
||||
for (const vv of v) lines.push(`${k}: ${vv}`);
|
||||
} else {
|
||||
lines.push(`${k}: ${v}`);
|
||||
}
|
||||
}
|
||||
lines.push('', '');
|
||||
upstream.write(lines.join('\r\n'));
|
||||
|
||||
if (head?.length) upstream.write(head);
|
||||
|
||||
upstream.on('error', () => {
|
||||
try {
|
||||
socket.destroy();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
});
|
||||
socket.on('error', () => {
|
||||
try {
|
||||
upstream.destroy();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
});
|
||||
|
||||
upstream.pipe(socket);
|
||||
socket.pipe(upstream);
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
if (req.method === 'GET' && (req.url === '/healthz' || req.url?.startsWith('/healthz?'))) {
|
||||
send(
|
||||
res,
|
||||
200,
|
||||
{ 'content-type': 'application/json; charset=utf-8' },
|
||||
JSON.stringify({ ok: true, version: APP_VERSION, buildTimestamp: BUILD_TIMESTAMP, startedAt: STARTED_AT })
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||
if (isGraphqlPath(url.pathname)) {
|
||||
if (req.method === 'OPTIONS') {
|
||||
withCors(res);
|
||||
res.statusCode = 204;
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') {
|
||||
const user = resolveAuthenticatedUser(req);
|
||||
if (!user) {
|
||||
withCors(res);
|
||||
unauthorized(res);
|
||||
return;
|
||||
}
|
||||
}
|
||||
withCors(res);
|
||||
proxyGraphqlHttp(req, res);
|
||||
return;
|
||||
}
|
||||
if (req.method === 'GET' && url.pathname === '/whoami') {
|
||||
sendJson(res, 200, { ok: true, user: resolveAuthenticatedUser(req), mode: AUTH_MODE });
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === 'GET' && url.pathname === '/prefs/dlob-source') {
|
||||
const set = url.searchParams.get('set');
|
||||
if (!set) {
|
||||
sendJson(res, 200, { ok: true, dlobSource: resolveDlobSource(req) });
|
||||
return;
|
||||
}
|
||||
|
||||
const ok = setDlobSourceCookie(res, { secure: isHttpsRequest(req), dlobSource: set });
|
||||
if (!ok) {
|
||||
sendJson(res, 400, { ok: false, error: 'invalid_dlob_source' });
|
||||
return;
|
||||
}
|
||||
|
||||
res.statusCode = 302;
|
||||
res.setHeader('location', safeRedirectPath(url.searchParams.get('redirect') || '/'));
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === 'POST' && url.pathname === '/auth/login') {
|
||||
if (AUTH_MODE === 'off' || AUTH_MODE === 'none' || AUTH_MODE === 'disabled') {
|
||||
sendJson(res, 400, { ok: false, error: 'auth_disabled' });
|
||||
return;
|
||||
}
|
||||
|
||||
const raw = await readBody(req);
|
||||
const ct = String(req.headers['content-type'] || '').toLowerCase();
|
||||
let username = '';
|
||||
let password = '';
|
||||
if (ct.includes('application/json')) {
|
||||
let json;
|
||||
try {
|
||||
json = JSON.parse(raw);
|
||||
} catch {
|
||||
sendJson(res, 400, { ok: false, error: 'bad_json' });
|
||||
return;
|
||||
}
|
||||
username = typeof json?.username === 'string' ? json.username.trim() : '';
|
||||
password = typeof json?.password === 'string' ? json.password : '';
|
||||
} else {
|
||||
const params = new URLSearchParams(raw);
|
||||
username = String(params.get('username') || '').trim();
|
||||
password = String(params.get('password') || '');
|
||||
}
|
||||
|
||||
if (!username || !password || username.length > 64 || password.length > 200) {
|
||||
sendJson(res, 400, { ok: false, error: 'invalid_input' });
|
||||
return;
|
||||
}
|
||||
|
||||
const ok = verifyWithHtpasswd(username, password);
|
||||
if (!ok) {
|
||||
unauthorized(res);
|
||||
return;
|
||||
}
|
||||
|
||||
const secure = isHttpsRequest(req);
|
||||
setSessionCookie(res, secure, username);
|
||||
sendJson(res, 200, { ok: true, user: username });
|
||||
return;
|
||||
}
|
||||
|
||||
if ((req.method === 'POST' || req.method === 'GET') && (url.pathname === '/auth/logout' || url.pathname === '/logout')) {
|
||||
clearSessionCookie(res, isHttpsRequest(req));
|
||||
if (req.method === 'GET') {
|
||||
res.statusCode = 302;
|
||||
res.setHeader('location', '/');
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
sendJson(res, 200, { ok: true });
|
||||
return;
|
||||
}
|
||||
|
||||
if (BASIC_AUTH_ENABLED) {
|
||||
let creds;
|
||||
try {
|
||||
creds = loadBasicAuth();
|
||||
} catch (e) {
|
||||
send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isAuthorized(req, creds)) {
|
||||
basicAuthRequired(res);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
serveStatic(req, res);
|
||||
}
|
||||
|
||||
const server = http.createServer((req, res) => {
|
||||
handler(req, res).catch((e) => {
|
||||
if (res.headersSent) {
|
||||
res.destroy();
|
||||
return;
|
||||
}
|
||||
send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e));
|
||||
});
|
||||
});
|
||||
server.on('upgrade', (req, socket, head) => {
|
||||
try {
|
||||
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||
if (!isGraphqlPath(url.pathname)) {
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') {
|
||||
const user = resolveAuthenticatedUser(req);
|
||||
if (!user) {
|
||||
try {
|
||||
socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
}
|
||||
proxyGraphqlWs(req, socket, head);
|
||||
} catch {
|
||||
socket.destroy();
|
||||
}
|
||||
});
|
||||
server.listen(PORT, () => {
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
service: 'trade-frontend',
|
||||
port: PORT,
|
||||
staticDir: STATIC_DIR,
|
||||
hasuraUpstream: HASURA_UPSTREAM,
|
||||
basicAuthFile: BASIC_AUTH_FILE,
|
||||
basicAuthMode: BASIC_AUTH_MODE,
|
||||
authUserHeader: AUTH_USER_HEADER,
|
||||
authMode: AUTH_MODE,
|
||||
htpasswdFile: HTPASSWD_FILE,
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
});
|
||||
@@ -6,5 +6,16 @@ namespace: trade-prod
|
||||
resources:
|
||||
- ../../base
|
||||
|
||||
patchesStrategicMerge:
|
||||
- frontend-graphql-proxy-patch.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: trade-frontend-server-script
|
||||
files:
|
||||
- frontend-server.mjs
|
||||
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
|
||||
commonLabels:
|
||||
env: prod
|
||||
|
||||
18
kustomize/overlays/staging/frontend-graphql-proxy-patch.yaml
Normal file
18
kustomize/overlays/staging/frontend-graphql-proxy-patch.yaml
Normal file
@@ -0,0 +1,18 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: trade-frontend
|
||||
spec:
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: frontend
|
||||
volumeMounts:
|
||||
- name: frontend-server-script
|
||||
mountPath: /app/services/frontend/server.mjs
|
||||
subPath: frontend-server.mjs
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: frontend-server-script
|
||||
configMap:
|
||||
name: trade-frontend-server-script
|
||||
24
kustomize/overlays/staging/frontend-ingress-root.yaml
Normal file
24
kustomize/overlays/staging/frontend-ingress-root.yaml
Normal file
@@ -0,0 +1,24 @@
|
||||
apiVersion: networking.k8s.io/v1
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: trade-frontend-root
|
||||
annotations:
|
||||
cert-manager.io/cluster-issuer: letsencrypt-prod
|
||||
traefik.ingress.kubernetes.io/router.entrypoints: websecure
|
||||
spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- mpabi.pl
|
||||
secretName: mpabi-pl-tls
|
||||
rules:
|
||||
- host: mpabi.pl
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
pathType: Prefix
|
||||
backend:
|
||||
service:
|
||||
name: trade-frontend
|
||||
port:
|
||||
number: 8081
|
||||
@@ -9,10 +9,10 @@ spec:
|
||||
ingressClassName: traefik
|
||||
tls:
|
||||
- hosts:
|
||||
- trade.rv32i.pl
|
||||
secretName: trade-rv32i-pl-tls
|
||||
- trade.mpabi.pl
|
||||
secretName: trade-mpabi-pl-tls
|
||||
rules:
|
||||
- host: trade.rv32i.pl
|
||||
- host: trade.mpabi.pl
|
||||
http:
|
||||
paths:
|
||||
- path: /
|
||||
|
||||
732
kustomize/overlays/staging/frontend-server.mjs
Normal file
732
kustomize/overlays/staging/frontend-server.mjs
Normal file
@@ -0,0 +1,732 @@
|
||||
import crypto from 'node:crypto';
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import fs from 'node:fs';
|
||||
import http from 'node:http';
|
||||
import https from 'node:https';
|
||||
import net from 'node:net';
|
||||
import path from 'node:path';
|
||||
import tls from 'node:tls';
|
||||
|
||||
const PORT = Number.parseInt(process.env.PORT || '8081', 10);
|
||||
if (!Number.isInteger(PORT) || PORT <= 0) throw new Error(`Invalid PORT: ${process.env.PORT}`);
|
||||
|
||||
const APP_VERSION = String(process.env.APP_VERSION || 'v1').trim() || 'v1';
|
||||
const BUILD_TIMESTAMP = String(process.env.BUILD_TIMESTAMP || '').trim() || undefined;
|
||||
const STARTED_AT = new Date().toISOString();
|
||||
|
||||
const STATIC_DIR = process.env.STATIC_DIR || '/srv';
|
||||
const BASIC_AUTH_FILE = process.env.BASIC_AUTH_FILE || '/tokens/frontend.json';
|
||||
const HASURA_UPSTREAM = process.env.HASURA_UPSTREAM || 'http://hasura:8080';
|
||||
const HASURA_GRAPHQL_PATH = process.env.HASURA_GRAPHQL_PATH || '/v1/graphql';
|
||||
const GRAPHQL_CORS_ORIGIN = process.env.GRAPHQL_CORS_ORIGIN || process.env.CORS_ORIGIN || '*';
|
||||
const BASIC_AUTH_MODE = String(process.env.BASIC_AUTH_MODE || 'on')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const BASIC_AUTH_ENABLED = !['off', 'false', '0', 'disabled', 'none'].includes(BASIC_AUTH_MODE);
|
||||
const AUTH_USER_HEADER = String(process.env.AUTH_USER_HEADER || 'x-trade-user')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const AUTH_MODE = String(process.env.AUTH_MODE || 'session')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const HTPASSWD_FILE = String(process.env.HTPASSWD_FILE || '/auth/users').trim();
|
||||
const AUTH_SESSION_SECRET_FILE = String(process.env.AUTH_SESSION_SECRET_FILE || '').trim() || null;
|
||||
const AUTH_SESSION_COOKIE = String(process.env.AUTH_SESSION_COOKIE || 'trade_session')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const AUTH_SESSION_TTL_SECONDS = Number.parseInt(process.env.AUTH_SESSION_TTL_SECONDS || '43200', 10); // 12h
|
||||
const DLOB_SOURCE_COOKIE = String(process.env.DLOB_SOURCE_COOKIE || 'trade_dlob_source').trim() || 'trade_dlob_source';
|
||||
const DLOB_SOURCE_DEFAULT = String(process.env.DLOB_SOURCE_DEFAULT || 'mevnode').trim() || 'mevnode';
|
||||
|
||||
function readJson(filePath) {
|
||||
const raw = fs.readFileSync(filePath, 'utf8');
|
||||
return JSON.parse(raw);
|
||||
}
|
||||
|
||||
function readText(filePath) {
|
||||
return fs.readFileSync(filePath, 'utf8');
|
||||
}
|
||||
|
||||
function timingSafeEqualStr(a, b) {
|
||||
const aa = Buffer.from(String(a), 'utf8');
|
||||
const bb = Buffer.from(String(b), 'utf8');
|
||||
if (aa.length !== bb.length) return false;
|
||||
return crypto.timingSafeEqual(aa, bb);
|
||||
}
|
||||
|
||||
function timingSafeEqualBuf(a, b) {
|
||||
if (!(a instanceof Uint8Array) || !(b instanceof Uint8Array)) return false;
|
||||
if (a.length !== b.length) return false;
|
||||
return crypto.timingSafeEqual(Buffer.from(a), Buffer.from(b));
|
||||
}
|
||||
|
||||
function loadBasicAuth() {
|
||||
const j = readJson(BASIC_AUTH_FILE);
|
||||
const username = (j?.username || '').toString();
|
||||
const password = (j?.password || '').toString();
|
||||
if (!username || !password) throw new Error(`Invalid BASIC_AUTH_FILE: ${BASIC_AUTH_FILE}`);
|
||||
return { username, password };
|
||||
}
|
||||
|
||||
function send(res, status, headers, body) {
|
||||
res.statusCode = status;
|
||||
for (const [k, v] of Object.entries(headers || {})) res.setHeader(k, v);
|
||||
if (body == null) return void res.end();
|
||||
res.end(body);
|
||||
}
|
||||
|
||||
function sendJson(res, status, body) {
|
||||
send(res, status, { 'content-type': 'application/json; charset=utf-8', 'cache-control': 'no-store' }, JSON.stringify(body));
|
||||
}
|
||||
|
||||
function basicAuthRequired(res) {
|
||||
res.setHeader('www-authenticate', 'Basic realm="trade"');
|
||||
send(res, 401, { 'content-type': 'text/plain; charset=utf-8' }, 'unauthorized');
|
||||
}
|
||||
|
||||
function unauthorized(res) {
|
||||
sendJson(res, 401, { ok: false, error: 'unauthorized' });
|
||||
}
|
||||
|
||||
function isAuthorized(req, creds) {
|
||||
const auth = req.headers.authorization || '';
|
||||
const m = String(auth).match(/^Basic\s+(.+)$/i);
|
||||
if (!m?.[1]) return false;
|
||||
let decoded;
|
||||
try {
|
||||
decoded = Buffer.from(m[1], 'base64').toString('utf8');
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
const idx = decoded.indexOf(':');
|
||||
if (idx < 0) return false;
|
||||
const u = decoded.slice(0, idx);
|
||||
const p = decoded.slice(idx + 1);
|
||||
return timingSafeEqualStr(u, creds.username) && timingSafeEqualStr(p, creds.password);
|
||||
}
|
||||
|
||||
const MIME = {
|
||||
'.html': 'text/html; charset=utf-8',
|
||||
'.css': 'text/css; charset=utf-8',
|
||||
'.js': 'application/javascript; charset=utf-8',
|
||||
'.mjs': 'application/javascript; charset=utf-8',
|
||||
'.json': 'application/json; charset=utf-8',
|
||||
'.svg': 'image/svg+xml',
|
||||
'.png': 'image/png',
|
||||
'.jpg': 'image/jpeg',
|
||||
'.jpeg': 'image/jpeg',
|
||||
'.gif': 'image/gif',
|
||||
'.ico': 'image/x-icon',
|
||||
'.txt': 'text/plain; charset=utf-8',
|
||||
'.map': 'application/json; charset=utf-8',
|
||||
};
|
||||
|
||||
function contentTypeFor(filePath) {
|
||||
return MIME[path.extname(filePath).toLowerCase()] || 'application/octet-stream';
|
||||
}
|
||||
|
||||
function safePathFromUrlPath(urlPath) {
|
||||
const decoded = decodeURIComponent(urlPath);
|
||||
const cleaned = decoded.replace(/\0/g, '');
|
||||
// strip leading slash so join() doesn't ignore STATIC_DIR
|
||||
const rel = cleaned.replace(/^\/+/, '');
|
||||
const normalized = path.normalize(rel);
|
||||
// prevent traversal
|
||||
if (normalized.startsWith('..') || path.isAbsolute(normalized)) return null;
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function injectIndexHtml(html, { dlobSource, redirectPath }) {
|
||||
const src = normalizeDlobSource(dlobSource) || 'mevnode';
|
||||
const redirect = safeRedirectPath(redirectPath);
|
||||
const hrefBase = `/prefs/dlob-source?redirect=${encodeURIComponent(redirect)}&set=`;
|
||||
|
||||
const styleActive = 'font-weight:700;text-decoration:underline;';
|
||||
const styleInactive = 'font-weight:400;text-decoration:none;';
|
||||
|
||||
const snippet = `
|
||||
<!-- trade: dlob source switch -->
|
||||
<div style="position:fixed;right:12px;bottom:12px;z-index:2147483647;background:rgba(0,0,0,0.72);color:#fff;padding:8px 10px;border-radius:10px;font:12px/1.2 system-ui,-apple-system,Segoe UI,Roboto,sans-serif;backdrop-filter:blur(6px);">
|
||||
<span style="opacity:0.85;margin-right:6px;">DLOB</span>
|
||||
<a href="${hrefBase}mevnode" style="color:#fff;${src === 'mevnode' ? styleActive : styleInactive}">mevnode</a>
|
||||
<span style="opacity:0.6;margin:0 6px;">|</span>
|
||||
<a href="${hrefBase}drift" style="color:#fff;${src === 'drift' ? styleActive : styleInactive}">drift</a>
|
||||
</div>
|
||||
`;
|
||||
|
||||
const bodyClose = /<\/body>/i;
|
||||
if (bodyClose.test(html)) return html.replace(bodyClose, `${snippet}</body>`);
|
||||
return `${html}\n${snippet}\n`;
|
||||
}
|
||||
|
||||
function serveStatic(req, res) {
|
||||
if (req.method !== 'GET' && req.method !== 'HEAD') {
|
||||
send(res, 405, { 'content-type': 'text/plain; charset=utf-8' }, 'method_not_allowed');
|
||||
return;
|
||||
}
|
||||
|
||||
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||
const rel = safePathFromUrlPath(url.pathname);
|
||||
if (rel == null) {
|
||||
send(res, 400, { 'content-type': 'text/plain; charset=utf-8' }, 'bad_request');
|
||||
return;
|
||||
}
|
||||
|
||||
const root = path.resolve(STATIC_DIR);
|
||||
const fileCandidate = path.resolve(root, rel);
|
||||
if (!fileCandidate.startsWith(root)) {
|
||||
send(res, 400, { 'content-type': 'text/plain; charset=utf-8' }, 'bad_request');
|
||||
return;
|
||||
}
|
||||
|
||||
const trySend = (filePath) => {
|
||||
try {
|
||||
const st = fs.statSync(filePath);
|
||||
if (st.isDirectory()) return trySend(path.join(filePath, 'index.html'));
|
||||
res.statusCode = 200;
|
||||
res.setHeader('content-type', contentTypeFor(filePath));
|
||||
res.setHeader('cache-control', filePath.endsWith('index.html') ? 'no-cache' : 'public, max-age=31536000');
|
||||
if (req.method === 'HEAD') return void res.end();
|
||||
if (filePath.endsWith('index.html')) {
|
||||
const html = fs.readFileSync(filePath, 'utf8');
|
||||
const injected = injectIndexHtml(html, {
|
||||
dlobSource: resolveDlobSource(req),
|
||||
redirectPath: url.pathname + url.search,
|
||||
});
|
||||
res.end(injected);
|
||||
return true;
|
||||
}
|
||||
fs.createReadStream(filePath).pipe(res);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
|
||||
// exact file, otherwise SPA fallback
|
||||
if (trySend(fileCandidate)) return;
|
||||
const indexPath = path.join(root, 'index.html');
|
||||
if (trySend(indexPath)) return;
|
||||
|
||||
send(res, 404, { 'content-type': 'text/plain; charset=utf-8' }, 'not_found');
|
||||
}
|
||||
|
||||
function stripHopByHopHeaders(headers) {
|
||||
const hop = new Set([
|
||||
'connection',
|
||||
'keep-alive',
|
||||
'proxy-authenticate',
|
||||
'proxy-authorization',
|
||||
'te',
|
||||
'trailer',
|
||||
'transfer-encoding',
|
||||
'upgrade',
|
||||
]);
|
||||
const out = {};
|
||||
for (const [k, v] of Object.entries(headers || {})) {
|
||||
if (hop.has(k.toLowerCase())) continue;
|
||||
out[k] = v;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function readHeader(req, name) {
|
||||
const v = req.headers[String(name).toLowerCase()];
|
||||
return Array.isArray(v) ? v[0] : v;
|
||||
}
|
||||
|
||||
function readCookie(req, name) {
|
||||
const raw = typeof req.headers.cookie === 'string' ? req.headers.cookie : '';
|
||||
if (!raw) return null;
|
||||
const needle = `${name}=`;
|
||||
for (const part of raw.split(';')) {
|
||||
const t = part.trim();
|
||||
if (!t.startsWith(needle)) continue;
|
||||
return t.slice(needle.length) || null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function normalizeDlobSource(value) {
|
||||
const v = String(value ?? '')
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (v === 'mevnode') return 'mevnode';
|
||||
if (v === 'drift') return 'drift';
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolveDlobSource(req) {
|
||||
const fromCookie = normalizeDlobSource(readCookie(req, DLOB_SOURCE_COOKIE));
|
||||
if (fromCookie) return fromCookie;
|
||||
return normalizeDlobSource(DLOB_SOURCE_DEFAULT) || 'mevnode';
|
||||
}
|
||||
|
||||
function safeRedirectPath(value) {
|
||||
const s = String(value ?? '').trim();
|
||||
if (!s.startsWith('/')) return '/';
|
||||
if (s.startsWith('//')) return '/';
|
||||
return s.replace(/\r|\n/g, '');
|
||||
}
|
||||
|
||||
function setDlobSourceCookie(res, { secure, dlobSource }) {
|
||||
const src = normalizeDlobSource(dlobSource);
|
||||
if (!src) return false;
|
||||
const parts = [
|
||||
`${DLOB_SOURCE_COOKIE}=${src}`,
|
||||
'Path=/',
|
||||
'SameSite=Lax',
|
||||
'HttpOnly',
|
||||
'Max-Age=31536000',
|
||||
];
|
||||
if (secure) parts.push('Secure');
|
||||
res.setHeader('set-cookie', parts.join('; '));
|
||||
return true;
|
||||
}
|
||||
|
||||
function resolveAuthUser(req) {
|
||||
const user = readHeader(req, AUTH_USER_HEADER) || readHeader(req, 'x-webauth-user');
|
||||
const value = typeof user === 'string' ? user.trim() : '';
|
||||
return value || null;
|
||||
}
|
||||
|
||||
function isHttpsRequest(req) {
|
||||
const xf = readHeader(req, 'x-forwarded-proto');
|
||||
if (typeof xf === 'string' && xf.toLowerCase() === 'https') return true;
|
||||
return Boolean(req.socket && req.socket.encrypted);
|
||||
}
|
||||
|
||||
function base64urlEncode(buf) {
|
||||
return Buffer.from(buf)
|
||||
.toString('base64')
|
||||
.replace(/\+/g, '-')
|
||||
.replace(/\//g, '_')
|
||||
.replace(/=+$/g, '');
|
||||
}
|
||||
|
||||
function base64urlDecode(str) {
|
||||
const cleaned = String(str).replace(/-/g, '+').replace(/_/g, '/');
|
||||
const pad = cleaned.length % 4 === 0 ? '' : '='.repeat(4 - (cleaned.length % 4));
|
||||
return Buffer.from(cleaned + pad, 'base64');
|
||||
}
|
||||
|
||||
function loadSessionSecret() {
|
||||
if (process.env.AUTH_SESSION_SECRET && String(process.env.AUTH_SESSION_SECRET).trim()) {
|
||||
return Buffer.from(String(process.env.AUTH_SESSION_SECRET).trim(), 'utf8');
|
||||
}
|
||||
if (AUTH_SESSION_SECRET_FILE) {
|
||||
try {
|
||||
const txt = readText(AUTH_SESSION_SECRET_FILE).trim();
|
||||
if (txt) return Buffer.from(txt, 'utf8');
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
return crypto.randomBytes(32);
|
||||
}
|
||||
|
||||
const SESSION_SECRET = loadSessionSecret();
|
||||
|
||||
function signSessionPayload(payloadB64) {
|
||||
return crypto.createHmac('sha256', SESSION_SECRET).update(payloadB64).digest();
|
||||
}
|
||||
|
||||
function makeSessionCookieValue(username) {
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const exp = now + (Number.isFinite(AUTH_SESSION_TTL_SECONDS) && AUTH_SESSION_TTL_SECONDS > 0 ? AUTH_SESSION_TTL_SECONDS : 43200);
|
||||
const payload = JSON.stringify({ u: String(username), exp });
|
||||
const payloadB64 = base64urlEncode(Buffer.from(payload, 'utf8'));
|
||||
const sigB64 = base64urlEncode(signSessionPayload(payloadB64));
|
||||
return `${payloadB64}.${sigB64}`;
|
||||
}
|
||||
|
||||
function getSessionUser(req) {
|
||||
const raw = readCookie(req, AUTH_SESSION_COOKIE);
|
||||
if (!raw) return null;
|
||||
const parts = raw.split('.');
|
||||
if (parts.length !== 2) return null;
|
||||
const [payloadB64, sigB64] = parts;
|
||||
if (!payloadB64 || !sigB64) return null;
|
||||
|
||||
let payload;
|
||||
try {
|
||||
payload = JSON.parse(base64urlDecode(payloadB64).toString('utf8'));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
const u = typeof payload?.u === 'string' ? payload.u.trim() : '';
|
||||
const exp = Number(payload?.exp);
|
||||
if (!u || !Number.isFinite(exp)) return null;
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
if (now >= exp) return null;
|
||||
|
||||
const expected = signSessionPayload(payloadB64);
|
||||
let got;
|
||||
try {
|
||||
got = base64urlDecode(sigB64);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!timingSafeEqualBuf(expected, got)) return null;
|
||||
|
||||
return u;
|
||||
}
|
||||
|
||||
function resolveAuthenticatedUser(req) {
|
||||
const sessionUser = getSessionUser(req);
|
||||
if (sessionUser) return sessionUser;
|
||||
const headerUser = resolveAuthUser(req);
|
||||
if (headerUser) return headerUser;
|
||||
if (AUTH_MODE === 'off' || AUTH_MODE === 'none' || AUTH_MODE === 'disabled') return 'anonymous';
|
||||
return null;
|
||||
}
|
||||
|
||||
function clearSessionCookie(res, secure) {
|
||||
const parts = [`${AUTH_SESSION_COOKIE}=`, 'Path=/', 'Max-Age=0', 'HttpOnly', 'SameSite=Lax'];
|
||||
if (secure) parts.push('Secure');
|
||||
res.setHeader('set-cookie', parts.join('; '));
|
||||
}
|
||||
|
||||
function setSessionCookie(res, secure, username) {
|
||||
const value = makeSessionCookieValue(username);
|
||||
const parts = [
|
||||
`${AUTH_SESSION_COOKIE}=${value}`,
|
||||
'Path=/',
|
||||
`Max-Age=${Number.isFinite(AUTH_SESSION_TTL_SECONDS) ? AUTH_SESSION_TTL_SECONDS : 43200}`,
|
||||
'HttpOnly',
|
||||
'SameSite=Lax',
|
||||
];
|
||||
if (secure) parts.push('Secure');
|
||||
res.setHeader('set-cookie', parts.join('; '));
|
||||
}
|
||||
|
||||
function verifyWithHtpasswd(username, password) {
|
||||
try {
|
||||
const r = spawnSync('htpasswd', ['-vb', HTPASSWD_FILE, String(username), String(password)], {
|
||||
stdio: 'ignore',
|
||||
timeout: 3000,
|
||||
});
|
||||
return r.status === 0;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readBody(req, limitBytes = 1024 * 16) {
|
||||
return new Promise((resolve, reject) => {
|
||||
let total = 0;
|
||||
const chunks = [];
|
||||
req.on('data', (chunk) => {
|
||||
total += chunk.length;
|
||||
if (total > limitBytes) {
|
||||
reject(new Error('payload_too_large'));
|
||||
req.destroy();
|
||||
return;
|
||||
}
|
||||
chunks.push(chunk);
|
||||
});
|
||||
req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8')));
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function withCors(res) {
|
||||
res.setHeader('access-control-allow-origin', GRAPHQL_CORS_ORIGIN);
|
||||
res.setHeader('access-control-allow-methods', 'GET,POST,OPTIONS');
|
||||
res.setHeader(
|
||||
'access-control-allow-headers',
|
||||
'content-type, authorization, x-hasura-admin-secret, x-hasura-role, x-hasura-user-id, x-hasura-dlob-source'
|
||||
);
|
||||
}
|
||||
|
||||
function proxyGraphqlHttp(req, res) {
|
||||
const upstreamBase = new URL(HASURA_UPSTREAM);
|
||||
const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||
|
||||
const target = new URL(upstreamBase.toString());
|
||||
target.pathname = HASURA_GRAPHQL_PATH;
|
||||
target.search = inUrl.search;
|
||||
|
||||
const isHttps = target.protocol === 'https:';
|
||||
const lib = isHttps ? https : http;
|
||||
|
||||
const headers = stripHopByHopHeaders(req.headers);
|
||||
headers.host = target.host;
|
||||
delete headers['x-hasura-dlob-source'];
|
||||
headers['x-hasura-dlob-source'] = resolveDlobSource(req);
|
||||
|
||||
const upstreamReq = lib.request(
|
||||
{
|
||||
protocol: target.protocol,
|
||||
hostname: target.hostname,
|
||||
port: target.port || (isHttps ? 443 : 80),
|
||||
method: req.method,
|
||||
path: target.pathname + target.search,
|
||||
headers,
|
||||
},
|
||||
(upstreamRes) => {
|
||||
const outHeaders = stripHopByHopHeaders(upstreamRes.headers);
|
||||
withCors(res);
|
||||
res.writeHead(upstreamRes.statusCode || 502, outHeaders);
|
||||
upstreamRes.pipe(res);
|
||||
}
|
||||
);
|
||||
|
||||
upstreamReq.on('error', (err) => {
|
||||
if (!res.headersSent) {
|
||||
withCors(res);
|
||||
send(res, 502, { 'content-type': 'text/plain; charset=utf-8' }, `bad_gateway: ${err?.message || err}`);
|
||||
} else {
|
||||
res.destroy();
|
||||
}
|
||||
});
|
||||
|
||||
req.pipe(upstreamReq);
|
||||
}
|
||||
|
||||
function isGraphqlPath(pathname) {
|
||||
return pathname === '/graphql' || pathname === '/graphql-ws';
|
||||
}
|
||||
|
||||
function proxyGraphqlWs(req, socket, head) {
|
||||
const upstreamBase = new URL(HASURA_UPSTREAM);
|
||||
const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||
|
||||
const target = new URL(upstreamBase.toString());
|
||||
target.pathname = HASURA_GRAPHQL_PATH;
|
||||
target.search = inUrl.search;
|
||||
|
||||
const port = Number(target.port || (target.protocol === 'https:' ? 443 : 80));
|
||||
const host = target.hostname;
|
||||
|
||||
const connect =
|
||||
target.protocol === 'https:'
|
||||
? () => tls.connect({ host, port, servername: host })
|
||||
: () => net.connect({ host, port });
|
||||
|
||||
const upstream = connect();
|
||||
upstream.setNoDelay(true);
|
||||
socket.setNoDelay(true);
|
||||
|
||||
// For WebSocket upgrades we must forward `connection`/`upgrade` and related headers.
|
||||
const headers = { ...req.headers };
|
||||
delete headers['content-length'];
|
||||
delete headers['content-type'];
|
||||
headers.host = target.host;
|
||||
delete headers['x-hasura-dlob-source'];
|
||||
headers['x-hasura-dlob-source'] = resolveDlobSource(req);
|
||||
|
||||
const lines = [];
|
||||
lines.push(`GET ${target.pathname + target.search} HTTP/1.1`);
|
||||
for (const [k, v] of Object.entries(headers)) {
|
||||
if (v == null) continue;
|
||||
if (Array.isArray(v)) {
|
||||
for (const vv of v) lines.push(`${k}: ${vv}`);
|
||||
} else {
|
||||
lines.push(`${k}: ${v}`);
|
||||
}
|
||||
}
|
||||
lines.push('', '');
|
||||
upstream.write(lines.join('\r\n'));
|
||||
|
||||
if (head?.length) upstream.write(head);
|
||||
|
||||
upstream.on('error', () => {
|
||||
try {
|
||||
socket.destroy();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
});
|
||||
socket.on('error', () => {
|
||||
try {
|
||||
upstream.destroy();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
});
|
||||
|
||||
upstream.pipe(socket);
|
||||
socket.pipe(upstream);
|
||||
}
|
||||
|
||||
async function handler(req, res) {
|
||||
if (req.method === 'GET' && (req.url === '/healthz' || req.url?.startsWith('/healthz?'))) {
|
||||
send(
|
||||
res,
|
||||
200,
|
||||
{ 'content-type': 'application/json; charset=utf-8' },
|
||||
JSON.stringify({ ok: true, version: APP_VERSION, buildTimestamp: BUILD_TIMESTAMP, startedAt: STARTED_AT })
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||
if (isGraphqlPath(url.pathname)) {
|
||||
if (req.method === 'OPTIONS') {
|
||||
withCors(res);
|
||||
res.statusCode = 204;
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') {
|
||||
const user = resolveAuthenticatedUser(req);
|
||||
if (!user) {
|
||||
withCors(res);
|
||||
unauthorized(res);
|
||||
return;
|
||||
}
|
||||
}
|
||||
withCors(res);
|
||||
proxyGraphqlHttp(req, res);
|
||||
return;
|
||||
}
|
||||
if (req.method === 'GET' && url.pathname === '/whoami') {
|
||||
sendJson(res, 200, { ok: true, user: resolveAuthenticatedUser(req), mode: AUTH_MODE });
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === 'GET' && url.pathname === '/prefs/dlob-source') {
|
||||
const set = url.searchParams.get('set');
|
||||
if (!set) {
|
||||
sendJson(res, 200, { ok: true, dlobSource: resolveDlobSource(req) });
|
||||
return;
|
||||
}
|
||||
|
||||
const ok = setDlobSourceCookie(res, { secure: isHttpsRequest(req), dlobSource: set });
|
||||
if (!ok) {
|
||||
sendJson(res, 400, { ok: false, error: 'invalid_dlob_source' });
|
||||
return;
|
||||
}
|
||||
|
||||
res.statusCode = 302;
|
||||
res.setHeader('location', safeRedirectPath(url.searchParams.get('redirect') || '/'));
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
if (req.method === 'POST' && url.pathname === '/auth/login') {
|
||||
if (AUTH_MODE === 'off' || AUTH_MODE === 'none' || AUTH_MODE === 'disabled') {
|
||||
sendJson(res, 400, { ok: false, error: 'auth_disabled' });
|
||||
return;
|
||||
}
|
||||
|
||||
const raw = await readBody(req);
|
||||
const ct = String(req.headers['content-type'] || '').toLowerCase();
|
||||
let username = '';
|
||||
let password = '';
|
||||
if (ct.includes('application/json')) {
|
||||
let json;
|
||||
try {
|
||||
json = JSON.parse(raw);
|
||||
} catch {
|
||||
sendJson(res, 400, { ok: false, error: 'bad_json' });
|
||||
return;
|
||||
}
|
||||
username = typeof json?.username === 'string' ? json.username.trim() : '';
|
||||
password = typeof json?.password === 'string' ? json.password : '';
|
||||
} else {
|
||||
const params = new URLSearchParams(raw);
|
||||
username = String(params.get('username') || '').trim();
|
||||
password = String(params.get('password') || '');
|
||||
}
|
||||
|
||||
if (!username || !password || username.length > 64 || password.length > 200) {
|
||||
sendJson(res, 400, { ok: false, error: 'invalid_input' });
|
||||
return;
|
||||
}
|
||||
|
||||
const ok = verifyWithHtpasswd(username, password);
|
||||
if (!ok) {
|
||||
unauthorized(res);
|
||||
return;
|
||||
}
|
||||
|
||||
const secure = isHttpsRequest(req);
|
||||
setSessionCookie(res, secure, username);
|
||||
sendJson(res, 200, { ok: true, user: username });
|
||||
return;
|
||||
}
|
||||
|
||||
if ((req.method === 'POST' || req.method === 'GET') && (url.pathname === '/auth/logout' || url.pathname === '/logout')) {
|
||||
clearSessionCookie(res, isHttpsRequest(req));
|
||||
if (req.method === 'GET') {
|
||||
res.statusCode = 302;
|
||||
res.setHeader('location', '/');
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
sendJson(res, 200, { ok: true });
|
||||
return;
|
||||
}
|
||||
|
||||
if (BASIC_AUTH_ENABLED) {
|
||||
let creds;
|
||||
try {
|
||||
creds = loadBasicAuth();
|
||||
} catch (e) {
|
||||
send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e));
|
||||
return;
|
||||
}
|
||||
|
||||
if (!isAuthorized(req, creds)) {
|
||||
basicAuthRequired(res);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
serveStatic(req, res);
|
||||
}
|
||||
|
||||
const server = http.createServer((req, res) => {
|
||||
handler(req, res).catch((e) => {
|
||||
if (res.headersSent) {
|
||||
res.destroy();
|
||||
return;
|
||||
}
|
||||
send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e));
|
||||
});
|
||||
});
|
||||
server.on('upgrade', (req, socket, head) => {
|
||||
try {
|
||||
const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`);
|
||||
if (!isGraphqlPath(url.pathname)) {
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') {
|
||||
const user = resolveAuthenticatedUser(req);
|
||||
if (!user) {
|
||||
try {
|
||||
socket.write('HTTP/1.1 401 Unauthorized\r\n\r\n');
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
socket.destroy();
|
||||
return;
|
||||
}
|
||||
}
|
||||
proxyGraphqlWs(req, socket, head);
|
||||
} catch {
|
||||
socket.destroy();
|
||||
}
|
||||
});
|
||||
server.listen(PORT, () => {
|
||||
console.log(
|
||||
JSON.stringify(
|
||||
{
|
||||
service: 'trade-frontend',
|
||||
port: PORT,
|
||||
staticDir: STATIC_DIR,
|
||||
hasuraUpstream: HASURA_UPSTREAM,
|
||||
basicAuthFile: BASIC_AUTH_FILE,
|
||||
basicAuthMode: BASIC_AUTH_MODE,
|
||||
authUserHeader: AUTH_USER_HEADER,
|
||||
authMode: AUTH_MODE,
|
||||
htpasswdFile: HTPASSWD_FILE,
|
||||
},
|
||||
null,
|
||||
2
|
||||
)
|
||||
);
|
||||
});
|
||||
@@ -12,3 +12,5 @@ spec:
|
||||
value: "true"
|
||||
- name: HASURA_GRAPHQL_DEV_MODE
|
||||
value: "true"
|
||||
- name: HASURA_GRAPHQL_CORS_DOMAIN
|
||||
value: "http://localhost:5173,http://127.0.0.1:5173,https://trade.mpabi.pl"
|
||||
|
||||
@@ -7,10 +7,20 @@ resources:
|
||||
- ../../base
|
||||
- pgadmin.yaml
|
||||
- frontend-ingress.yaml
|
||||
- frontend-ingress-root.yaml
|
||||
|
||||
patchesStrategicMerge:
|
||||
- hasura-patch.yaml
|
||||
- frontend-auth-patch.yaml
|
||||
- frontend-graphql-proxy-patch.yaml
|
||||
|
||||
configMapGenerator:
|
||||
- name: trade-frontend-server-script
|
||||
files:
|
||||
- frontend-server.mjs
|
||||
|
||||
generatorOptions:
|
||||
disableNameSuffixHash: true
|
||||
|
||||
commonLabels:
|
||||
env: staging
|
||||
|
||||
Reference in New Issue
Block a user