diff --git a/kustomize/base/api/deployment.yaml b/kustomize/base/api/deployment.yaml deleted file mode 100644 index a1e299c..0000000 --- a/kustomize/base/api/deployment.yaml +++ /dev/null @@ -1,77 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trade-api -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: trade-api - template: - metadata: - labels: - app.kubernetes.io/name: trade-api - spec: - imagePullSecrets: - - name: gitea-registry - volumes: - - name: trade-api-wrapper - configMap: - name: trade-api-wrapper - - name: trade-api-upstream - configMap: - name: trade-api-upstream - containers: - - name: api - image: gitea.mpabi.pl/trade/trade-api:k3s-20260111095435 - imagePullPolicy: IfNotPresent - command: ["node", "/override/wrapper.mjs"] - ports: - - name: http - containerPort: 8787 - volumeMounts: - - name: trade-api-wrapper - mountPath: /override/wrapper.mjs - subPath: wrapper.mjs - readOnly: true - - name: trade-api-upstream - mountPath: /app/services/api/server.mjs - subPath: server.mjs - readOnly: true - env: - - name: PORT - value: "8787" - - name: UPSTREAM_PORT - value: "8788" - - name: UPSTREAM_ENTRY - value: "/app/services/api/server.mjs" - - name: UPSTREAM_HOST - value: "127.0.0.1" - - name: APP_VERSION - value: "staging" - - name: HASURA_GRAPHQL_URL - value: "http://hasura:8080/v1/graphql" - - name: HASURA_ADMIN_SECRET - valueFrom: - secretKeyRef: - name: trade-hasura - key: HASURA_GRAPHQL_ADMIN_SECRET - - name: API_ADMIN_SECRET - valueFrom: - secretKeyRef: - name: trade-api - key: API_ADMIN_SECRET - - name: CORS_ORIGIN - value: "*" - readinessProbe: - httpGet: - path: /healthz - port: http - initialDelaySeconds: 2 - periodSeconds: 5 - livenessProbe: - httpGet: - path: /healthz - port: http - initialDelaySeconds: 10 - periodSeconds: 10 diff --git a/kustomize/base/api/server.mjs b/kustomize/base/api/server.mjs deleted file mode 100644 index a2e2547..0000000 --- a/kustomize/base/api/server.mjs +++ /dev/null @@ -1,1759 +0,0 @@ -import crypto from 'node:crypto'; -import fs from 'node:fs'; -import http from 'node:http'; -import path from 'node:path'; -import { fileURLToPath } from 'node:url'; - -const SCRIPT_DIR = path.dirname(fileURLToPath(import.meta.url)); -const PROJECT_ROOT = path.resolve(SCRIPT_DIR, '..', '..'); -const TOKENS_DIR = path.join(PROJECT_ROOT, 'tokens'); - -function readJsonFile(filePath) { - try { - const raw = fs.readFileSync(filePath, 'utf8'); - return JSON.parse(raw); - } catch { - return undefined; - } -} - -function getIsoNow() { - return new Date().toISOString(); -} - -function base64Url(buf) { - return Buffer.from(buf) - .toString('base64') - .replace(/=/g, '') - .replace(/\+/g, '-') - .replace(/\//g, '_'); -} - -function sha256Hex(text) { - return crypto.createHash('sha256').update(text, 'utf8').digest('hex'); -} - -function generateToken() { - return `alg_${base64Url(crypto.randomBytes(32))}`; -} - -function getHeader(req, name) { - const v = req.headers[String(name).toLowerCase()]; - return Array.isArray(v) ? v[0] : v; -} - -function readBearerToken(req) { - const auth = getHeader(req, 'authorization'); - if (auth && typeof auth === 'string') { - const m = auth.match(/^Bearer\s+(.+)$/i); - if (m && m[1]) return m[1].trim(); - } - const apiKey = getHeader(req, 'x-api-key'); - if (apiKey && typeof apiKey === 'string' && apiKey.trim()) return apiKey.trim(); - return undefined; -} - -function withCors(res, corsOrigin) { - res.setHeader('access-control-allow-origin', corsOrigin); - res.setHeader('access-control-allow-methods', 'GET,POST,OPTIONS'); - res.setHeader( - 'access-control-allow-headers', - 'content-type, authorization, x-api-key, x-admin-secret' - ); -} - -function sendJson(res, status, body, corsOrigin) { - withCors(res, corsOrigin); - res.statusCode = status; - res.setHeader('content-type', 'application/json; charset=utf-8'); - res.end(JSON.stringify(body)); -} - -async function readBodyJson(req, { maxBytes }) { - const chunks = []; - let total = 0; - for await (const chunk of req) { - total += chunk.length; - if (total > maxBytes) throw new Error('payload_too_large'); - chunks.push(chunk); - } - const text = Buffer.concat(chunks).toString('utf8'); - if (!text.trim()) return {}; - try { - return JSON.parse(text); - } catch { - throw new Error('invalid_json'); - } -} - -function normalizeGraphqlName(value, fallback, label) { - const v = String(value || fallback || '') - .trim() - .toLowerCase(); - if (!v) return String(fallback || ''); - if (!/^[a-z][a-z0-9_]*$/.test(v)) throw new Error(`Invalid ${label}: ${value}`); - return v; -} - -function resolveConfig() { - const hasuraTokens = readJsonFile(path.join(TOKENS_DIR, 'hasura.json')) || {}; - const apiTokens = readJsonFile(path.join(TOKENS_DIR, 'api.json')) || {}; - - const portRaw = process.env.PORT || process.env.API_PORT || apiTokens.port || '8787'; - const port = Number.parseInt(String(portRaw), 10); - if (!Number.isInteger(port) || port <= 0) throw new Error(`Invalid PORT: ${portRaw}`); - - const hasuraUrl = - process.env.HASURA_GRAPHQL_URL || - hasuraTokens.graphqlUrl || - hasuraTokens.apiUrl || - 'http://localhost:8080/v1/graphql'; - - const hasuraAdminSecret = - process.env.HASURA_ADMIN_SECRET || hasuraTokens.adminSecret || hasuraTokens.hasuraAdminSecret; - - const apiAdminSecret = process.env.API_ADMIN_SECRET || apiTokens.adminSecret; - - const corsOrigin = process.env.CORS_ORIGIN || apiTokens.corsOrigin || '*'; - - const appVersion = String(process.env.APP_VERSION || 'v1').trim() || 'v1'; - const buildTimestamp = String(process.env.BUILD_TIMESTAMP || '').trim() || undefined; - const startedAt = getIsoNow(); - - const ticksTable = normalizeGraphqlName(process.env.TICKS_TABLE, 'drift_ticks', 'TICKS_TABLE'); - const candlesFunction = normalizeGraphqlName( - process.env.CANDLES_FUNCTION, - 'get_drift_candles', - 'CANDLES_FUNCTION' - ); - - return { - port, - hasuraUrl, - hasuraAdminSecret, - apiAdminSecret, - corsOrigin, - appVersion, - buildTimestamp, - startedAt, - ticksTable, - candlesFunction, - }; -} - -async function hasuraRequest(cfg, { admin }, query, variables) { - const headers = { 'content-type': 'application/json' }; - if (admin) { - if (!cfg.hasuraAdminSecret) throw new Error('Missing HASURA_ADMIN_SECRET (or tokens/hasura.json adminSecret)'); - headers['x-hasura-admin-secret'] = cfg.hasuraAdminSecret; - } - - const res = await fetch(cfg.hasuraUrl, { - method: 'POST', - headers, - body: JSON.stringify({ query, variables }), - }); - - const text = await res.text(); - if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`); - - let json; - try { - json = JSON.parse(text); - } catch { - throw new Error(`Hasura invalid JSON: ${text}`); - } - if (json.errors?.length) { - throw new Error(json.errors.map((e) => e.message).join(' | ')); - } - return json.data; -} - -async function readSolPriceUsd(cfg) { - // Best-effort: use DLOB stats for SOL-PERP if available, else latest tick mark/oracle. - try { - const q = ` - query SolPrice { - dlob_stats_latest(where: {market_name: {_eq: "SOL-PERP"}}, limit: 1) { - mid_price - mark_price - oracle_price - } - } - `; - const data = await hasuraRequest(cfg, { admin: true }, q, {}); - const row = data?.dlob_stats_latest?.[0]; - const p = parseNumeric(row?.mid_price) ?? parseNumeric(row?.mark_price) ?? parseNumeric(row?.oracle_price); - if (p != null && p > 0) return p; - } catch { - // ignore - } - - try { - const table = cfg.ticksTable; - const q = ` - query SolTick($limit: Int!) { - ${table}(where: {symbol: {_eq: "SOL-PERP"}}, order_by: {ts: desc}, limit: $limit) { - mark_price - oracle_price - } - } - `; - const data = await hasuraRequest(cfg, { admin: true }, q, { limit: 1 }); - const row = data?.[table]?.[0]; - const p = parseNumeric(row?.mark_price) ?? parseNumeric(row?.oracle_price); - if (p != null && p > 0) return p; - } catch { - // ignore - } - - return null; -} - -function normalizeScopes(value) { - if (!value) return []; - if (Array.isArray(value)) return value.map((v) => String(v)).filter(Boolean); - if (typeof value === 'string') { - return value - .split(',') - .map((s) => s.trim()) - .filter(Boolean); - } - return []; -} - -async function requireValidToken(cfg, req, requiredScope) { - const token = readBearerToken(req); - if (!token) return { ok: false, status: 401, error: 'missing_token' }; - - const hash = sha256Hex(token); - const query = ` - query ValidToken($hash: String!) { - api_tokens(where: {token_hash: {_eq: $hash}, revoked_at: {_is_null: true}}, limit: 1) { - id - name - scopes - } - } - `; - - let data; - try { - data = await hasuraRequest(cfg, { admin: true }, query, { hash }); - } catch (err) { - return { ok: false, status: 500, error: String(err?.message || err) }; - } - - const row = data?.api_tokens?.[0]; - if (!row?.id) return { ok: false, status: 401, error: 'invalid_or_revoked_token' }; - const scopes = normalizeScopes(row.scopes); - if (requiredScope && !scopes.includes(requiredScope)) { - return { ok: false, status: 403, error: 'missing_scope' }; - } - - // best-effort touch - const touch = ` - mutation TouchToken($id: uuid!, $ts: timestamptz!) { - update_api_tokens_by_pk(pk_columns: {id: $id}, _set: {last_used_at: $ts}) { id } - } - `; - hasuraRequest(cfg, { admin: true }, touch, { id: row.id, ts: new Date().toISOString() }).catch(() => {}); - - return { ok: true, token: { id: row.id, name: row.name } }; -} - -function toNumericString(value, fieldName) { - if (value == null) throw new Error(`invalid_${fieldName}`); - if (typeof value === 'number') { - if (!Number.isFinite(value)) throw new Error(`invalid_${fieldName}`); - return String(value); - } - if (typeof value === 'string') { - const s = value.trim(); - if (!s) throw new Error(`invalid_${fieldName}`); - const n = Number(s); - if (!Number.isFinite(n)) throw new Error(`invalid_${fieldName}`); - return s; - } - // best-effort: allow bigint-like or BN-like objects - if (typeof value?.toString === 'function') { - const s = String(value.toString()).trim(); - if (!s) throw new Error(`invalid_${fieldName}`); - const n = Number(s); - if (!Number.isFinite(n)) throw new Error(`invalid_${fieldName}`); - return s; - } - throw new Error(`invalid_${fieldName}`); -} - -function normalizeTick(input, tokenInfo) { - const ts = (input?.ts || input?.timestamp || getIsoNow())?.toString?.(); - const market_index = input?.market_index ?? input?.marketIndex; - const symbol = input?.symbol; - const oracle_price = input?.oracle_price ?? input?.oraclePrice ?? input?.price; - const mark_price = input?.mark_price ?? input?.markPrice ?? input?.mark; - const oracle_slot = input?.oracle_slot ?? input?.oracleSlot ?? input?.slot; - const source = (input?.source || 'api')?.toString?.(); - const raw = input?.raw && typeof input.raw === 'object' ? input.raw : undefined; - - if (!ts || Number.isNaN(Date.parse(ts))) throw new Error('invalid_ts'); - if (!Number.isInteger(market_index)) throw new Error('invalid_market_index'); - if (typeof symbol !== 'string' || !symbol.trim()) throw new Error('invalid_symbol'); - const oracleStr = toNumericString(oracle_price, 'oracle_price'); - const markStr = mark_price == null ? undefined : toNumericString(mark_price, 'mark_price'); - - const slotNum = - oracle_slot == null ? undefined : Number.isFinite(Number(oracle_slot)) ? Number(oracle_slot) : undefined; - - const mergedRaw = - raw || tokenInfo - ? { - ...(raw || {}), - ingestedBy: tokenInfo ? { tokenId: tokenInfo.id, name: tokenInfo.name } : undefined, - } - : undefined; - - return { - ts, - market_index, - symbol: symbol.trim(), - // Postgres columns are NUMERIC; Hasura `numeric` scalar returns strings and expects string inputs. - oracle_price: oracleStr, - mark_price: markStr, - oracle_slot: slotNum, - source, - raw: mergedRaw, - }; -} - -async function insertTick(cfg, tick) { - const table = cfg.ticksTable; - const insertField = `insert_${table}_one`; - const mutation = ` - mutation InsertTick($object: ${table}_insert_input!) { - ${insertField}(object: $object) { - id - } - } - `; - - const data = await hasuraRequest(cfg, { admin: true }, mutation, { object: tick }); - return data?.[insertField]?.id; -} - -async function createApiToken(cfg, name, scopes, meta) { - const mutation = ` - mutation CreateToken($name: String!, $hash: String!, $scopes: [String!]!, $meta: jsonb) { - insert_api_tokens_one(object: {name: $name, token_hash: $hash, scopes: $scopes, meta: $meta}) { - id - name - created_at - } - } - `; - - for (let attempt = 0; attempt < 5; attempt++) { - const token = generateToken(); - const hash = sha256Hex(token); - try { - const data = await hasuraRequest(cfg, { admin: true }, mutation, { name, hash, scopes, meta }); - const row = data?.insert_api_tokens_one; - if (!row?.id) throw new Error('token_insert_failed'); - return { token, row }; - } catch (err) { - const msg = String(err?.message || err); - const isUniqueConflict = msg.toLowerCase().includes('unique') || msg.toLowerCase().includes('constraint'); - if (!isUniqueConflict) throw err; - } - } - throw new Error('token_generation_failed'); -} - -async function revokeApiToken(cfg, id) { - const mutation = ` - mutation RevokeToken($id: uuid!, $ts: timestamptz!) { - update_api_tokens_by_pk(pk_columns: {id: $id}, _set: {revoked_at: $ts}) { - id - revoked_at - } - } - `; - const data = await hasuraRequest(cfg, { admin: true }, mutation, { id, ts: new Date().toISOString() }); - return data?.update_api_tokens_by_pk?.id; -} - -function clampInt(value, min, max) { - const n = Number.parseInt(String(value), 10); - if (!Number.isFinite(n) || !Number.isInteger(n)) return min; - return Math.min(max, Math.max(min, n)); -} - -function clampNumber(value, min, max, fallback) { - const n = typeof value === 'number' ? value : Number(value); - if (!Number.isFinite(n)) return fallback; - return Math.min(max, Math.max(min, n)); -} - -function parseNumeric(value) { - if (value == null) return null; - if (typeof value === 'number') return Number.isFinite(value) ? value : null; - if (typeof value === 'string') { - const s = value.trim(); - if (!s) return null; - const n = Number(s); - return Number.isFinite(n) ? n : null; - } - return null; -} - -function tsToUnixSeconds(value) { - if (typeof value === 'number') return Number.isFinite(value) ? value : null; - if (typeof value !== 'string') return null; - const ms = Date.parse(value); - if (!Number.isFinite(ms)) return null; - return Math.floor(ms / 1000); -} - -function isUuid(value) { - const s = String(value || '').trim(); - return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(s); -} - -function getByPath(obj, pathStr) { - if (!obj || typeof obj !== 'object') return undefined; - const parts = String(pathStr || '').split('.').filter(Boolean); - let cur = obj; - for (const p of parts) { - if (!cur || typeof cur !== 'object') return undefined; - cur = cur[p]; - } - return cur; -} - -function readNumberFromPayload(payload, paths) { - for (const p of paths) { - const v = getByPath(payload, p); - const n = parseNumeric(v); - if (n != null) return n; - } - return null; -} - -function readTextFromPayload(payload, paths) { - for (const p of paths) { - const v = getByPath(payload, p); - if (typeof v === 'string' && v.trim()) return v.trim(); - } - return null; -} - -function inferContractSizeUsd(contract) { - return ( - readNumberFromPayload(contract, [ - 'desired.size_usd', - 'desired.sizeUsd', - 'desired.notional_usd', - 'desired.notionalUsd', - 'entry.size_usd', - 'entry.sizeUsd', - 'entry.notional_usd', - 'entry.notionalUsd', - 'entry.order_intent.size_usd', - 'entry.order_intent.sizeUsd', - 'desired.order_intent.size_usd', - 'desired.order_intent.sizeUsd', - ]) || null - ); -} - -function inferContractSide(contract) { - const raw = - readTextFromPayload(contract, [ - 'desired.side', - 'entry.side', - 'entry.order_intent.side', - 'desired.order_intent.side', - 'desired.direction', - 'entry.direction', - ]) || ''; - const v = raw.toLowerCase(); - if (v === 'long' || v === 'buy') return 'long'; - if (v === 'short' || v === 'sell') return 'short'; - return null; -} - -function sumCostsFromEvents(events) { - const totals = { - tradeFeeUsd: 0, - txFeeUsd: 0, - slippageUsd: 0, - fundingUsd: 0, - realizedPnlUsd: 0, - txCount: 0, - fillCount: 0, - cancelCount: 0, - modifyCount: 0, - errorCount: 0, - }; - - for (const ev of events || []) { - const t = String(ev?.event_type || '').toLowerCase(); - const payload = ev?.payload && typeof ev.payload === 'object' ? ev.payload : {}; - - const tradeFeeUsd = - readNumberFromPayload(payload, ['realized_fee_usd', 'trade_fee_usd', 'fee_usd', 'fees.trade_fee_usd', 'fees.usd']) || 0; - const txFeeUsd = - readNumberFromPayload(payload, ['realized_tx_usd', 'tx_fee_usd', 'network_fee_usd', 'fees.tx_fee_usd', 'fees.network_usd']) || 0; - const slippageUsd = - readNumberFromPayload(payload, ['slippage_usd', 'realized_slippage_usd', 'execution_usd', 'realized_execution_usd']) || 0; - const fundingUsd = readNumberFromPayload(payload, ['funding_usd', 'realized_funding_usd']) || 0; - const pnlUsd = readNumberFromPayload(payload, ['realized_pnl_usd', 'pnl_usd']) || 0; - const txCount = readNumberFromPayload(payload, ['tx_count', 'txCount']) || 0; - - totals.tradeFeeUsd += tradeFeeUsd; - totals.txFeeUsd += txFeeUsd; - totals.slippageUsd += slippageUsd; - totals.fundingUsd += fundingUsd; - totals.realizedPnlUsd += pnlUsd; - totals.txCount += txCount; - - if (t.includes('fill')) totals.fillCount += 1; - if (t.includes('cancel')) totals.cancelCount += 1; - if (t.includes('modify') || t.includes('reprice')) totals.modifyCount += 1; - if (t.includes('error') || String(ev?.severity || '').toLowerCase() === 'error') totals.errorCount += 1; - } - - const totalCostsUsd = totals.tradeFeeUsd + totals.txFeeUsd + totals.slippageUsd + totals.fundingUsd; - - return { - ...totals, - totalCostsUsd, - netPnlUsd: totals.realizedPnlUsd - totalCostsUsd, - }; -} - -function buildCostSeriesFromEvents(events, { maxPoints }) { - const points = []; - const totals = { - tradeFeeUsd: 0, - txFeeUsd: 0, - slippageUsd: 0, - fundingUsd: 0, - realizedPnlUsd: 0, - }; - - for (const ev of events || []) { - const ts = ev?.ts; - if (!ts) continue; - const payload = ev?.payload && typeof ev.payload === 'object' ? ev.payload : {}; - - const tradeFeeUsd = - readNumberFromPayload(payload, ['realized_fee_usd', 'trade_fee_usd', 'fee_usd', 'fees.trade_fee_usd', 'fees.usd']) || 0; - const txFeeUsd = - readNumberFromPayload(payload, ['realized_tx_usd', 'tx_fee_usd', 'network_fee_usd', 'fees.tx_fee_usd', 'fees.network_usd']) || 0; - const slippageUsd = - readNumberFromPayload(payload, ['slippage_usd', 'realized_slippage_usd', 'execution_usd', 'realized_execution_usd']) || 0; - const fundingUsd = readNumberFromPayload(payload, ['funding_usd', 'realized_funding_usd']) || 0; - const pnlUsd = readNumberFromPayload(payload, ['realized_pnl_usd', 'pnl_usd']) || 0; - - totals.tradeFeeUsd += tradeFeeUsd; - totals.txFeeUsd += txFeeUsd; - totals.slippageUsd += slippageUsd; - totals.fundingUsd += fundingUsd; - totals.realizedPnlUsd += pnlUsd; - - const totalCostsUsd = totals.tradeFeeUsd + totals.txFeeUsd + totals.slippageUsd + totals.fundingUsd; - points.push({ - ts, - tradeFeeUsd: totals.tradeFeeUsd, - txFeeUsd: totals.txFeeUsd, - slippageUsd: totals.slippageUsd, - fundingUsd: totals.fundingUsd, - totalCostsUsd, - realizedPnlUsd: totals.realizedPnlUsd, - netPnlUsd: totals.realizedPnlUsd - totalCostsUsd, - }); - } - - const cap = Math.max(50, Math.min(10_000, Number(maxPoints) || 600)); - if (points.length <= cap) return points; - - const step = Math.ceil(points.length / cap); - const sampled = []; - for (let i = 0; i < points.length; i += step) sampled.push(points[i]); - const last = points[points.length - 1]; - if (sampled[sampled.length - 1] !== last) sampled.push(last); - return sampled; -} - -function flowFromDelta(delta) { - if (delta > 0) return { up: 1, down: 0, flat: 0 }; - if (delta < 0) return { up: 0, down: 1, flat: 0 }; - return { up: 0, down: 0, flat: 1 }; -} - -function computeCandleFlowFromTicks({ candle, bucketSeconds, points, nowSec, isCurrent }) { - const start = candle.time; - const end = start + bucketSeconds; - const progressEnd = isCurrent ? Math.min(end, Math.max(start, nowSec)) : end; - - const totalWindow = progressEnd - start; - if (!(totalWindow > 0)) return flowFromDelta(candle.close - candle.open); - - let up = 0; - let down = 0; - let flat = 0; - - let prevT = start; - let prevP = candle.open; - - for (const pt of points || []) { - const t = clampNumber(pt.t, prevT, progressEnd, prevT); - const dt = t - prevT; - if (dt > 0) { - const delta = pt.p - prevP; - if (delta > 0) up += dt; - else if (delta < 0) down += dt; - else flat += dt; - prevT = t; - } - prevP = pt.p; - if (prevT >= progressEnd) break; - } - - const tail = progressEnd - prevT; - if (tail > 0) flat += tail; - - const sum = up + down + flat; - if (!(sum > 0)) return flowFromDelta(candle.close - candle.open); - return { up: up / sum, down: down / sum, flat: flat / sum }; -} - -function computeCandleFlowRowsFromTicks({ candle, bucketSeconds, points, rows, nowSec, isCurrent }) { - const start = candle.time; - const end = start + bucketSeconds; - const progressEnd = isCurrent ? Math.min(end, Math.max(start, nowSec)) : end; - const totalWindow = progressEnd - start; - if (!(totalWindow > 0)) { - const overall = candle.close - candle.open; - const dir = overall > 0 ? 1 : overall < 0 ? -1 : 0; - return new Array(rows).fill(dir); - } - - const rowDirs = new Array(rows).fill(0); - - const pts = [{ t: start, p: candle.open }, ...(points || []), { t: progressEnd, p: candle.close }]; - for (let i = 1; i < pts.length; i += 1) { - const a = pts[i - 1]; - const b = pts[i]; - const t1 = clampNumber(a.t, start, progressEnd, start); - const t2 = clampNumber(b.t, start, progressEnd, start); - if (!(t2 > t1)) continue; - - const delta = b.p - a.p; - const dir = delta > 0 ? 1 : delta < 0 ? -1 : 0; - - const from = Math.max(0, Math.min(rows - 1, Math.floor(((t1 - start) / bucketSeconds) * rows))); - const to = Math.max(0, Math.min(rows - 1, Math.floor(((t2 - start) / bucketSeconds) * rows))); - for (let r = from; r <= to; r += 1) rowDirs[r] = dir; - } - - return rowDirs; -} - -function computeCandleFlowSlicesFromTicks({ candle, bucketSeconds, points, rows, nowSec, isCurrent }) { - const start = candle.time; - const end = start + bucketSeconds; - const progressEnd = isCurrent ? Math.min(end, Math.max(start, nowSec)) : end; - const dirs = new Array(rows).fill(0); - const moves = new Array(rows).fill(0); - - const totalWindow = progressEnd - start; - if (!(totalWindow > 0)) { - const overall = candle.close - candle.open; - const dir = overall > 0 ? 1 : overall < 0 ? -1 : 0; - dirs.fill(dir); - return { dirs, moves }; - } - - const pts = [...(points || []), { t: progressEnd, p: candle.close }]; - let idx = 0; - let lastP = candle.open; - - for (let r = 0; r < rows; r += 1) { - const sliceStart = start + (r / rows) * bucketSeconds; - if (!(sliceStart < progressEnd)) break; - const sliceEnd = Math.min(progressEnd, start + ((r + 1) / rows) * bucketSeconds); - - while (idx < pts.length && pts[idx].t <= sliceStart) { - lastP = pts[idx].p; - idx += 1; - } - const pStart = lastP; - - while (idx < pts.length && pts[idx].t <= sliceEnd) { - lastP = pts[idx].p; - idx += 1; - } - const pEnd = lastP; - - const delta = pEnd - pStart; - const dir = delta > 0 ? 1 : delta < 0 ? -1 : 0; - const move = Math.abs(delta); - dirs[r] = dir; - moves[r] = Number.isFinite(move) ? move : 0; - } - - return { dirs, moves }; -} - -function computeCandleFlowMovesFromTicks({ candle, bucketSeconds, points, rows, nowSec, isCurrent }) { - const start = candle.time; - const end = start + bucketSeconds; - const progressEnd = isCurrent ? Math.min(end, Math.max(start, nowSec)) : end; - const totalWindow = progressEnd - start; - const out = new Array(rows).fill(0); - if (!(totalWindow > 0)) return out; - - const pts = [...(points || []), { t: progressEnd, p: candle.close }]; - let idx = 0; - let lastP = candle.open; - - for (let r = 0; r < rows; r += 1) { - const sliceStart = start + (r / rows) * bucketSeconds; - if (!(sliceStart < progressEnd)) break; - const sliceEnd = Math.min(progressEnd, start + ((r + 1) / rows) * bucketSeconds); - - while (idx < pts.length && pts[idx].t <= sliceStart) { - lastP = pts[idx].p; - idx += 1; - } - const pStart = lastP; - - while (idx < pts.length && pts[idx].t <= sliceEnd) { - lastP = pts[idx].p; - idx += 1; - } - const pEnd = lastP; - - const move = Math.abs(pEnd - pStart); - out[r] = Number.isFinite(move) ? move : 0; - } - - return out; -} - -function parseTimeframeToSeconds(tf) { - const v = String(tf || '').trim().toLowerCase(); - if (!v) return 60; - const m = v.match(/^(\d+)(s|m|h|d)$/); - if (!m) throw new Error(`invalid_tf`); - const num = Number.parseInt(m[1], 10); - if (!Number.isInteger(num) || num <= 0) throw new Error(`invalid_tf`); - const unit = m[2]; - const mult = unit === 's' ? 1 : unit === 'm' ? 60 : unit === 'h' ? 3600 : 86400; - return num * mult; -} - -function fillForwardCandles(candles, { bucketSeconds, limit, nowSec }) { - if (!Array.isArray(candles) || candles.length === 0) return []; - if (!Number.isFinite(bucketSeconds) || bucketSeconds <= 0) return candles; - if (!Number.isFinite(limit) || limit <= 0) return candles; - - // `candles` should be ascending by time. - const cleaned = candles - .filter((c) => c && Number.isFinite(c.time) && Number.isFinite(c.close)) - .slice() - .sort((a, b) => a.time - b.time); - - if (cleaned.length === 0) return []; - - const map = new Map(cleaned.map((c) => [c.time, c])); - const lastDataTime = cleaned[cleaned.length - 1].time; - const now = Number.isFinite(nowSec) ? nowSec : Math.floor(Date.now() / 1000); - const alignedNow = Math.floor(now / bucketSeconds) * bucketSeconds; - const endTime = Math.max(alignedNow, lastDataTime); - const startTime = endTime - bucketSeconds * (limit - 1); - - const baseline = cleaned[0]; - const out = []; - out.length = limit; - - let prev = null; - for (let i = 0; i < limit; i += 1) { - const t = startTime + i * bucketSeconds; - const hit = map.get(t); - if (hit) { - const c = { ...hit }; - c.volume = Number.isFinite(c.volume) ? c.volume : 0; - // Keep continuity: next candle opens where previous candle closed. - // This avoids visual "gaps" when ticks are sparse. - if (prev && Number.isFinite(prev.close)) { - const prevClose = Number(prev.close); - c.open = prevClose; - c.high = Math.max(Number(c.high), prevClose, Number(c.close)); - c.low = Math.min(Number(c.low), prevClose, Number(c.close)); - } - out[i] = c; - prev = c; - continue; - } - - const base = prev || baseline; - const close = Number(base.close); - const oracle = base.oracle == null ? null : Number(base.oracle); - - const filled = { - time: t, - open: close, - high: close, - low: close, - close, - volume: 0, - oracle: Number.isFinite(oracle) ? oracle : null, - }; - out[i] = filled; - prev = filled; - } - - return out.filter((c) => c && Number.isFinite(c.time) && Number.isFinite(c.open) && Number.isFinite(c.close)); -} - -function pickFlowPointBucketSeconds(bucketSeconds, rowsPerCandle) { - // We want a point step that is: - // - small enough to capture intra-candle direction, - // - but derived from already-cached candle buckets (1s/3s/5s/...). - const targetStep = Math.max(1, Math.floor(bucketSeconds / Math.max(1, rowsPerCandle))); - const candidates = [1, 3, 5, 15, 30, 60, 180, 300, 900, 1800, 3600, 14_400, 43_200, 86_400]; - let best = candidates[0]; - for (const c of candidates) { - if (c <= targetStep) best = c; - } - return best; -} - -function mean(values) { - if (!values.length) return 0; - return values.reduce((a, b) => a + b, 0) / values.length; -} - -function stddev(values) { - if (!values.length) return 0; - const m = mean(values); - const v = values.reduce((acc, x) => acc + (x - m) * (x - m), 0) / values.length; - return Math.sqrt(v); -} - -function sma(values, period) { - if (period <= 0) throw new Error('period must be > 0'); - const out = new Array(values.length).fill(null); - let sum = 0; - for (let i = 0; i < values.length; i++) { - sum += values[i]; - if (i >= period) sum -= values[i - period]; - if (i >= period - 1) out[i] = sum / period; - } - return out; -} - -function ema(values, period) { - if (period <= 0) throw new Error('period must be > 0'); - const out = new Array(values.length).fill(null); - const k = 2 / (period + 1); - if (values.length < period) return out; - const first = mean(values.slice(0, period)); - out[period - 1] = first; - let prev = first; - for (let i = period; i < values.length; i++) { - const next = values[i] * k + prev * (1 - k); - out[i] = next; - prev = next; - } - return out; -} - -function rsi(values, period) { - if (period <= 0) throw new Error('period must be > 0'); - const out = new Array(values.length).fill(null); - if (values.length <= period) return out; - - let gains = 0; - let losses = 0; - for (let i = 1; i <= period; i++) { - const change = values[i] - values[i - 1]; - if (change >= 0) gains += change; - else losses -= change; - } - let avgGain = gains / period; - let avgLoss = losses / period; - - const rs = avgLoss === 0 ? Number.POSITIVE_INFINITY : avgGain / avgLoss; - out[period] = 100 - 100 / (1 + rs); - - for (let i = period + 1; i < values.length; i++) { - const change = values[i] - values[i - 1]; - const gain = Math.max(change, 0); - const loss = Math.max(-change, 0); - avgGain = (avgGain * (period - 1) + gain) / period; - avgLoss = (avgLoss * (period - 1) + loss) / period; - const rs2 = avgLoss === 0 ? Number.POSITIVE_INFINITY : avgGain / avgLoss; - out[i] = 100 - 100 / (1 + rs2); - } - - return out; -} - -function bollingerBands(values, period, stdDevMult) { - if (period <= 0) throw new Error('period must be > 0'); - const upper = new Array(values.length).fill(null); - const lower = new Array(values.length).fill(null); - const mid = sma(values, period); - - for (let i = period - 1; i < values.length; i++) { - const window = values.slice(i - period + 1, i + 1); - const sd = stddev(window); - const m = mid[i]; - if (m == null) continue; - upper[i] = m + stdDevMult * sd; - lower[i] = m - stdDevMult * sd; - } - - return { upper, lower, mid }; -} - -function macd(values, fastPeriod = 12, slowPeriod = 26, signalPeriod = 9) { - const fast = ema(values, fastPeriod); - const slow = ema(values, slowPeriod); - const macdLine = values.map((_, i) => { - const f = fast[i]; - const s = slow[i]; - return f == null || s == null ? null : f - s; - }); - - const signal = new Array(values.length).fill(null); - const k = 2 / (signalPeriod + 1); - let seeded = false; - let prev = 0; - const buf = []; - - for (let i = 0; i < macdLine.length; i++) { - const v = macdLine[i]; - if (v == null) continue; - - if (!seeded) { - buf.push(v); - if (buf.length === signalPeriod) { - const first = mean(buf); - signal[i] = first; - prev = first; - seeded = true; - } - continue; - } - - const next = v * k + prev * (1 - k); - signal[i] = next; - prev = next; - } - - return { macd: macdLine, signal }; -} - -function toSeries(times, values) { - return times.map((t, i) => ({ time: t, value: values[i] ?? null })); -} - -function isAdmin(cfg, req) { - if (!cfg.apiAdminSecret) return false; - const provided = getHeader(req, 'x-admin-secret'); - return typeof provided === 'string' && provided === cfg.apiAdminSecret; -} - -async function handler(cfg, req, res) { - if (req.method === 'OPTIONS') { - withCors(res, cfg.corsOrigin); - res.statusCode = 204; - res.end(); - return; - } - - const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`); - const pathname = url.pathname; - - if (req.method === 'GET' && pathname === '/healthz') { - sendJson( - res, - 200, - { - ok: true, - version: cfg.appVersion, - buildTimestamp: cfg.buildTimestamp, - startedAt: cfg.startedAt, - ticksTable: cfg.ticksTable, - candlesFunction: cfg.candlesFunction, - }, - cfg.corsOrigin - ); - return; - } - - if (req.method === 'GET' && pathname === '/v1/chart') { - const auth = await requireValidToken(cfg, req, 'read'); - if (!auth.ok) { - sendJson(res, auth.status, { ok: false, error: auth.error }, cfg.corsOrigin); - return; - } - - const symbol = (url.searchParams.get('symbol') || '').trim(); - const source = (url.searchParams.get('source') || '').trim(); - const basisRaw = (url.searchParams.get('basis') || '').trim().toLowerCase(); - const tf = (url.searchParams.get('tf') || url.searchParams.get('timeframe') || '1m').trim(); - const limit = clampInt(url.searchParams.get('limit') || '300', 10, 2000); - - if (!symbol) { - sendJson(res, 400, { ok: false, error: 'missing_symbol' }, cfg.corsOrigin); - return; - } - - let bucketSeconds; - try { - bucketSeconds = parseTimeframeToSeconds(tf); - } catch { - sendJson(res, 400, { ok: false, error: 'invalid_tf' }, cfg.corsOrigin); - return; - } - - const sourceKey = source || ''; - const basis = basisRaw === 'mark' ? 'mark' : basisRaw === 'oracle' || !basisRaw ? 'oracle' : null; - if (!basis) { - sendJson(res, 400, { ok: false, error: 'invalid_basis' }, cfg.corsOrigin); - return; - } - - try { - // Cache-first: read precomputed candles from `drift_candles_cache`. - // Fallback: compute on-demand via `get_drift_candles()` if cache not warmed yet. - const qCache = ` - query CachedCandles($symbol: String!, $bucket: Int!, $limit: Int!, $source: String!) { - drift_candles_cache( - where: {symbol: {_eq: $symbol}, bucket_seconds: {_eq: $bucket}, source: {_eq: $source}} - order_by: {bucket: desc} - limit: $limit - ) { - bucket - open - high - low - close - oracle_open - oracle_high - oracle_low - oracle_close - ticks - } - } - `; - - const cacheData = await hasuraRequest(cfg, { admin: true }, qCache, { - symbol, - bucket: bucketSeconds, - limit, - source: sourceKey, - }); - - let rows = cacheData?.drift_candles_cache || []; - - if (!rows.length) { - const fn = cfg.candlesFunction; - const qFn = ` - query Candles($symbol: String!, $bucket: Int!, $limit: Int!, $source: String) { - ${fn}(args: { p_symbol: $symbol, p_bucket_seconds: $bucket, p_limit: $limit, p_source: $source }) { - bucket - open - high - low - close - oracle_open - oracle_high - oracle_low - oracle_close - ticks - } - } - `; - const data = await hasuraRequest(cfg, { admin: true }, qFn, { - symbol, - bucket: bucketSeconds, - limit, - source: source || null, - }); - rows = data?.[fn] || []; - } - - const nowSec = Math.floor(Date.now() / 1000); - - let candles = rows - .slice() - .reverse() - .map((r) => { - const time = Math.floor(Date.parse(r.bucket) / 1000); - const oracleClose = r.oracle_close == null ? null : Number(r.oracle_close); - const oracleOpen = r.oracle_open == null ? oracleClose : Number(r.oracle_open); - const oracleHigh = r.oracle_high == null ? oracleClose : Number(r.oracle_high); - const oracleLow = r.oracle_low == null ? oracleClose : Number(r.oracle_low); - - const markOpen = Number(r.open); - const markHigh = Number(r.high); - const markLow = Number(r.low); - const markClose = Number(r.close); - - const open = basis === 'oracle' ? oracleOpen : markOpen; - const high = basis === 'oracle' ? oracleHigh : markHigh; - const low = basis === 'oracle' ? oracleLow : markLow; - const close = basis === 'oracle' ? oracleClose : markClose; - - // Always expose oracle close (even if basis=mark). - const oracle = oracleClose; - const volume = Number(r.ticks || 0); - return { time, open, high, low, close, volume, oracle }; - }) - .filter((c) => Number.isFinite(c.time) && Number.isFinite(c.open) && Number.isFinite(c.close)); - - // Make candles continuous in time: if no tick happened in a bucket, emit a flat candle using last close. - // This keeps the chart stable for 1s/3s/... views and makes timeframe switching instant (cache + no gaps). - candles = fillForwardCandles(candles, { bucketSeconds, limit, nowSec }); - - // Flow = share of time spent moving up/down/flat inside each bucket. - // Used by the UI to render stacked volume bars describing microstructure. - const windowSeconds = bucketSeconds * candles.length; - const canComputeFlow = candles.length > 0 && windowSeconds > 0 && windowSeconds <= 86_400; // cap at 24h - const rowsPerCandle = Math.min(60, Math.max(12, Math.floor(bucketSeconds))); - const flowPointBucketSeconds = pickFlowPointBucketSeconds(bucketSeconds, rowsPerCandle); - - if (canComputeFlow) { - const firstStart = candles[0].time; - const lastStart = candles[candles.length - 1].time; - const fromIso = new Date(firstStart * 1000).toISOString(); - const toIso = new Date((lastStart + bucketSeconds) * 1000).toISOString(); - - try { - // Prefer flow computed from cached candles (fast, no raw tick scan). - // If cache is missing, fall back to a simple delta-based approximation. - const maxPoints = Math.min(86_400, Math.max(1_000, candles.length * rowsPerCandle * 2)); - - const q1s = ` - query FlowPts($symbol: String!, $source: String!, $bucketSeconds: Int!, $from: timestamptz!, $to: timestamptz!, $limit: Int!) { - drift_candles_cache( - where: { - symbol: {_eq: $symbol} - source: {_eq: $source} - bucket_seconds: {_eq: $bucketSeconds} - bucket: {_gte: $from, _lt: $to} - } - order_by: {bucket: asc} - limit: $limit - ) { - bucket - close - oracle_close - } - } - `; - - const pData = await hasuraRequest(cfg, { admin: true }, q1s, { - symbol, - source: sourceKey, - bucketSeconds: flowPointBucketSeconds, - from: fromIso, - to: toIso, - limit: maxPoints, - }); - - const ptsRows = pData?.drift_candles_cache || []; - const visibleStarts = new Set(candles.map((c) => c.time)); - const pointsByCandle = new Map(); - - for (const r of ptsRows) { - const t = tsToUnixSeconds(r.bucket); - if (t == null) continue; - const p = - basis === 'oracle' - ? parseNumeric(r.oracle_close) ?? parseNumeric(r.close) - : parseNumeric(r.close) ?? parseNumeric(r.oracle_close); - if (p == null) continue; - const idx = Math.floor((t - firstStart) / bucketSeconds); - const start = firstStart + idx * bucketSeconds; - if (!visibleStarts.has(start)) continue; - const list = pointsByCandle.get(start) || []; - list.push({ t, p }); - pointsByCandle.set(start, list); - } - - const lastCandleTime = candles[candles.length - 1]?.time ?? null; - for (const c of candles) { - const pts = pointsByCandle.get(c.time) || []; - const isCurrent = lastCandleTime != null && c.time === lastCandleTime; - c.flow = computeCandleFlowFromTicks({ candle: c, bucketSeconds, points: pts, nowSec, isCurrent }); - const slices = computeCandleFlowSlicesFromTicks({ - candle: c, - bucketSeconds, - points: pts, - rows: rowsPerCandle, - nowSec, - isCurrent, - }); - c.flowRows = slices.dirs; - c.flowMoves = slices.moves; - } - } catch { - for (const c of candles) { - const fallback = flowFromDelta(c.close - c.open); - c.flow = fallback; - const dir = fallback.up ? 1 : fallback.down ? -1 : 0; - c.flowRows = new Array(rowsPerCandle).fill(dir); - c.flowMoves = new Array(rowsPerCandle).fill(0); - } - } - } else { - for (const c of candles) { - const fallback = flowFromDelta(c.close - c.open); - c.flow = fallback; - const dir = fallback.up ? 1 : fallback.down ? -1 : 0; - c.flowRows = new Array(rowsPerCandle).fill(dir); - c.flowMoves = new Array(rowsPerCandle).fill(0); - } - } - - const times = candles.map((c) => c.time); - const closes = candles.map((c) => c.close); - const oracleSeries = toSeries(times, candles.map((c) => (c.oracle == null ? null : c.oracle))); - - const sma20 = toSeries(times, sma(closes, 20)); - const ema20 = toSeries(times, ema(closes, 20)); - const bb = bollingerBands(closes, 20, 2); - const bbUpper = toSeries(times, bb.upper); - const bbLower = toSeries(times, bb.lower); - const bbMid = toSeries(times, bb.mid); - const rsi14 = toSeries(times, rsi(closes, 14)); - const macdOut = macd(closes, 12, 26, 9); - const macdLine = toSeries(times, macdOut.macd); - const macdSignal = toSeries(times, macdOut.signal); - - sendJson( - res, - 200, - { - ok: true, - version: cfg.appVersion, - buildTimestamp: cfg.buildTimestamp, - ticksTable: cfg.ticksTable, - candlesFunction: cfg.candlesFunction, - symbol, - source: source || null, - basis, - tf, - bucketSeconds, - candles, - indicators: { - oracle: oracleSeries, - sma20, - ema20, - bb20: { upper: bbUpper, lower: bbLower, mid: bbMid }, - rsi14, - macd: { macd: macdLine, signal: macdSignal }, - }, - }, - cfg.corsOrigin - ); - } catch (err) { - sendJson(res, 500, { ok: false, error: String(err?.message || err) }, cfg.corsOrigin); - } - return; - } - - if (req.method === 'POST' && pathname === '/v1/ingest/tick') { - const auth = await requireValidToken(cfg, req, 'write'); - if (!auth.ok) { - sendJson(res, auth.status, { ok: false, error: auth.error }, cfg.corsOrigin); - return; - } - - let body; - try { - body = await readBodyJson(req, { maxBytes: 1024 * 1024 }); - } catch (err) { - const msg = String(err?.message || err); - if (msg === 'payload_too_large') { - sendJson(res, 413, { ok: false, error: 'payload_too_large' }, cfg.corsOrigin); - return; - } - sendJson(res, 400, { ok: false, error: 'invalid_json' }, cfg.corsOrigin); - return; - } - - let tick; - try { - tick = normalizeTick(body, auth.token); - } catch (err) { - sendJson(res, 400, { ok: false, error: String(err?.message || err) }, cfg.corsOrigin); - return; - } - - try { - const id = await insertTick(cfg, tick); - sendJson(res, 200, { ok: true, id }, cfg.corsOrigin); - } catch (err) { - sendJson(res, 500, { ok: false, error: String(err?.message || err) }, cfg.corsOrigin); - } - return; - } - - if (req.method === 'GET' && pathname === '/v1/ticks') { - const auth = await requireValidToken(cfg, req, 'read'); - if (!auth.ok) { - sendJson(res, auth.status, { ok: false, error: auth.error }, cfg.corsOrigin); - return; - } - - const symbol = url.searchParams.get('symbol') || ''; - const source = url.searchParams.get('source'); - const limitRaw = url.searchParams.get('limit') || '1000'; - const limit = Math.min(5000, Math.max(1, Number.parseInt(limitRaw, 10) || 1000)); - const from = url.searchParams.get('from'); - const to = url.searchParams.get('to'); - - if (!symbol.trim()) { - sendJson(res, 400, { ok: false, error: 'missing_symbol' }, cfg.corsOrigin); - return; - } - - const where = { symbol: { _eq: symbol.trim() } }; - if (source && source.trim()) where.source = { _eq: source.trim() }; - if (from || to) { - where.ts = {}; - if (from) where.ts._gte = from; - if (to) where.ts._lte = to; - } - - const table = cfg.ticksTable; - const query = ` - query Ticks($where: ${table}_bool_exp!, $limit: Int!) { - ${table}(where: $where, order_by: {ts: desc}, limit: $limit) { - ts - market_index - symbol - oracle_price - mark_price - oracle_slot - source - } - } - `; - - try { - const data = await hasuraRequest(cfg, { admin: true }, query, { where, limit }); - const ticks = (data?.[table] || []).slice().reverse(); - sendJson(res, 200, { ok: true, version: cfg.appVersion, ticksTable: cfg.ticksTable, ticks }, cfg.corsOrigin); - } catch (err) { - sendJson(res, 500, { ok: false, error: String(err?.message || err) }, cfg.corsOrigin); - } - return; - } - - // Contract monitoring + cost compute (read scope). - // This endpoint is meant for "live UI" polling/subscription aggregation on the backend. - // It is intentionally resilient to varying bot_events payload schemas. - if (req.method === 'GET' && pathname.startsWith('/v1/contracts/') && pathname.endsWith('/monitor')) { - const auth = await requireValidToken(cfg, req, 'read'); - if (!auth.ok) { - sendJson(res, auth.status, { ok: false, error: auth.error }, cfg.corsOrigin); - return; - } - - const parts = pathname.split('/').filter(Boolean); - const contractId = parts[2]; - if (!isUuid(contractId)) { - sendJson(res, 400, { ok: false, error: 'invalid_contract_id' }, cfg.corsOrigin); - return; - } - - const limit = clampInt(url.searchParams.get('eventsLimit') || '2000', 10, 50_000); - const wantSeries = (url.searchParams.get('series') || '').trim() === '1'; - const seriesMax = clampInt(url.searchParams.get('seriesMax') || '600', 50, 10_000); - - const qContract = ` - query ContractByPk($id: uuid!) { - bot_contracts_by_pk(id: $id) { - id - decision_id - bot_id - model_version - market_name - subaccount_id - status - desired - entry - manage - exit - gates - created_at - updated_at - last_heartbeat_at - ended_at - reason - } - } - `; - const qEvents = ` - query ContractEvents($id: uuid!, $limit: Int!) { - bot_events(where: {contract_id: {_eq: $id}}, order_by: {ts: asc}, limit: $limit) { - ts - contract_id - decision_id - bot_id - market_name - event_type - severity - payload - } - } - `; - - try { - const data = await hasuraRequest(cfg, { admin: true }, qContract, { id: contractId }); - const contract = data?.bot_contracts_by_pk; - if (!contract?.id) { - sendJson(res, 404, { ok: false, error: 'contract_not_found' }, cfg.corsOrigin); - return; - } - - const evData = await hasuraRequest(cfg, { admin: true }, qEvents, { id: contractId, limit }); - const events = evData?.bot_events || []; - const costs = sumCostsFromEvents(events); - const series = wantSeries ? buildCostSeriesFromEvents(events, { maxPoints: seriesMax }) : null; - - const sizeUsd = inferContractSizeUsd(contract); - const side = inferContractSide(contract); - - let closeEst = null; - if (contract.market_name && sizeUsd != null) { - const qSlip = ` - query Slippage($market: String!) { - dlob_slippage_latest_v2(where: {market_name: {_eq: $market}}) { - market_name - side - size_usd - mid_price - vwap_price - worst_price - impact_bps - fill_pct - updated_at - } - dlob_slippage_latest(where: {market_name: {_eq: $market}}) { - market_name - side - size_usd - mid_price - vwap_price - worst_price - impact_bps - fill_pct - updated_at - } - } - `; - const slipData = await hasuraRequest(cfg, { admin: true }, qSlip, { market: contract.market_name }); - const rowsV2 = slipData?.dlob_slippage_latest_v2 || []; - const rowsV1 = slipData?.dlob_slippage_latest || []; - const rows = rowsV2.length ? rowsV2 : rowsV1; - const pickNearest = (wantedSide) => { - const candidates = rows.filter((r) => String(r.side || '').toLowerCase() === wantedSide); - if (!candidates.length) return null; - let best = null; - let bestD = Infinity; - for (const r of candidates) { - const s = parseNumeric(r.size_usd); - if (s == null) continue; - const d = Math.abs(s - sizeUsd); - if (d < bestD) { - bestD = d; - best = r; - } - } - return best; - }; - - const buy = pickNearest('buy'); - const sell = pickNearest('sell'); - closeEst = { - requestedSizeUsd: sizeUsd, - entrySide: side, - suggestedCloseSide: side === 'long' ? 'sell' : side === 'short' ? 'buy' : null, - buy, - sell, - }; - } - - sendJson( - res, - 200, - { - ok: true, - contract, - eventsCount: events.length, - costs, - series, - closeEstimate: closeEst, - }, - cfg.corsOrigin - ); - } catch (err) { - sendJson(res, 500, { ok: false, error: String(err?.message || err) }, cfg.corsOrigin); - } - return; - } - - // Estimate costs for a new contract (read scope). - // Uses DLOB slippage table and simple fee/tx estimates (inputs). - if (req.method === 'POST' && pathname === '/v1/contracts/costs/estimate') { - const auth = await requireValidToken(cfg, req, 'read'); - if (!auth.ok) { - sendJson(res, auth.status, { ok: false, error: auth.error }, cfg.corsOrigin); - return; - } - - let body; - try { - body = await readBodyJson(req, { maxBytes: 256 * 1024 }); - } catch (err) { - const msg = String(err?.message || err); - if (msg === 'payload_too_large') { - sendJson(res, 413, { ok: false, error: 'payload_too_large' }, cfg.corsOrigin); - return; - } - sendJson(res, 400, { ok: false, error: 'invalid_json' }, cfg.corsOrigin); - return; - } - - const market = String(body?.market_name || body?.market || '').trim(); - if (!market) { - sendJson(res, 400, { ok: false, error: 'missing_market' }, cfg.corsOrigin); - return; - } - - const notionalUsd = parseNumeric(body?.notional_usd ?? body?.notionalUsd ?? body?.size_usd ?? body?.sizeUsd); - if (notionalUsd == null || !(notionalUsd > 0)) { - sendJson(res, 400, { ok: false, error: 'invalid_notional_usd' }, cfg.corsOrigin); - return; - } - - const sideRaw = String(body?.side || 'long').trim().toLowerCase(); - const entrySide = sideRaw === 'short' || sideRaw === 'sell' ? 'short' : 'long'; - const defaultTakerBps = parseNumeric(process.env.FEE_TAKER_BPS_DEFAULT) ?? 5; - const defaultMakerBps = parseNumeric(process.env.FEE_MAKER_BPS_DEFAULT) ?? 0; - const takerBps = clampNumber(parseNumeric(body?.fee_taker_bps) ?? defaultTakerBps, 0, 1000, defaultTakerBps); - const makerBps = clampNumber(parseNumeric(body?.fee_maker_bps) ?? defaultMakerBps, -1000, 1000, defaultMakerBps); - const orderType = String(body?.order_type || body?.orderType || 'market').trim().toLowerCase(); - const isMarket = orderType === 'market' || orderType === 'taker'; - const feeBps = isMarket ? takerBps : makerBps; - - let txFeeUsdEst = parseNumeric(body?.tx_fee_usd_est); - if (txFeeUsdEst == null) { - const baseLamports = parseNumeric(process.env.TX_BASE_FEE_LAMPORTS_EST) ?? 5000; - const sigs = parseNumeric(process.env.TX_SIGNATURES_EST) ?? 1; - const priorityLamports = parseNumeric(process.env.TX_PRIORITY_FEE_LAMPORTS_EST) ?? 0; - const lamports = Math.max(0, baseLamports) * Math.max(1, sigs) + Math.max(0, priorityLamports); - const sol = lamports / 1_000_000_000; - const solUsd = await readSolPriceUsd(cfg); - txFeeUsdEst = solUsd != null ? sol * solUsd : 0; - } - txFeeUsdEst = clampNumber(txFeeUsdEst, 0, 100, 0); - const defaultReprices = parseNumeric(process.env.EXPECTED_REPRICES_PER_ENTRY_DEFAULT) ?? 0; - const expectedReprices = clampInt(body?.expected_reprices_per_entry ?? body?.expectedReprices ?? String(defaultReprices), 0, 500); - const modifyTxCount = clampInt(body?.modify_tx_count ?? body?.modifyTxCount ?? '2', 0, 10); - - try { - const qSlip = ` - query Slippage($market: String!) { - dlob_slippage_latest_v2(where: {market_name: {_eq: $market}}) { - market_name - side - size_usd - mid_price - vwap_price - worst_price - impact_bps - fill_pct - updated_at - } - dlob_slippage_latest(where: {market_name: {_eq: $market}}) { - market_name - side - size_usd - mid_price - vwap_price - worst_price - impact_bps - fill_pct - updated_at - } - } - `; - const slipData = await hasuraRequest(cfg, { admin: true }, qSlip, { market }); - const rowsV2 = slipData?.dlob_slippage_latest_v2 || []; - const rowsV1 = slipData?.dlob_slippage_latest || []; - const rows = rowsV2.length ? rowsV2 : rowsV1; - const wantedSide = entrySide === 'long' ? 'buy' : 'sell'; - const candidates = rows.filter((r) => String(r.side || '').toLowerCase() === wantedSide); - let best = null; - let bestD = Infinity; - for (const r of candidates) { - const s = parseNumeric(r.size_usd); - if (s == null) continue; - const d = Math.abs(s - notionalUsd); - if (d < bestD) { - bestD = d; - best = r; - } - } - - const impactBps = parseNumeric(best?.impact_bps) ?? 0; - const slippageUsd = (notionalUsd * impactBps) / 10_000; - const tradeFeeUsd = (notionalUsd * feeBps) / 10_000; - const modifyCostUsd = expectedReprices * modifyTxCount * txFeeUsdEst; - const totalUsd = tradeFeeUsd + slippageUsd + txFeeUsdEst + modifyCostUsd; - const totalBps = (totalUsd / notionalUsd) * 10_000; - - sendJson( - res, - 200, - { - ok: true, - input: { - market_name: market, - notional_usd: notionalUsd, - side: entrySide, - order_type: orderType, - fee_bps: feeBps, - tx_fee_usd_est: txFeeUsdEst, - expected_reprices_per_entry: expectedReprices, - }, - dlob: best - ? { - size_usd: best.size_usd, - side: best.side, - mid_price: best.mid_price, - vwap_price: best.vwap_price, - impact_bps: best.impact_bps, - fill_pct: best.fill_pct, - updated_at: best.updated_at, - } - : null, - breakdown: { - trade_fee_usd: tradeFeeUsd, - slippage_usd: slippageUsd, - tx_fee_usd: txFeeUsdEst, - expected_modify_usd: modifyCostUsd, - total_usd: totalUsd, - total_bps: totalBps, - breakeven_bps: totalBps, - }, - }, - cfg.corsOrigin - ); - } catch (err) { - sendJson(res, 500, { ok: false, error: String(err?.message || err) }, cfg.corsOrigin); - } - return; - } - - if (req.method === 'POST' && pathname === '/v1/admin/tokens') { - if (!isAdmin(cfg, req)) { - sendJson(res, 401, { ok: false, error: 'admin_unauthorized' }, cfg.corsOrigin); - return; - } - - let body; - try { - body = await readBodyJson(req, { maxBytes: 1024 * 1024 }); - } catch { - sendJson(res, 400, { ok: false, error: 'invalid_json' }, cfg.corsOrigin); - return; - } - - const name = (body?.name || 'algo')?.toString?.().trim(); - if (!name) { - sendJson(res, 400, { ok: false, error: 'missing_name' }, cfg.corsOrigin); - return; - } - const scopes = normalizeScopes(body?.scopes); - const resolvedScopes = scopes.length ? scopes : ['write']; - - try { - const { token, row } = await createApiToken(cfg, name, resolvedScopes, body?.meta); - sendJson(res, 200, { ok: true, token, id: row.id, name: row.name, created_at: row.created_at }, cfg.corsOrigin); - } catch (err) { - sendJson(res, 500, { ok: false, error: String(err?.message || err) }, cfg.corsOrigin); - } - return; - } - - if (req.method === 'POST' && pathname === '/v1/admin/tokens/revoke') { - if (!isAdmin(cfg, req)) { - sendJson(res, 401, { ok: false, error: 'admin_unauthorized' }, cfg.corsOrigin); - return; - } - - let body; - try { - body = await readBodyJson(req, { maxBytes: 1024 * 1024 }); - } catch { - sendJson(res, 400, { ok: false, error: 'invalid_json' }, cfg.corsOrigin); - return; - } - - const id = (body?.id || '')?.toString?.().trim(); - if (!id) { - sendJson(res, 400, { ok: false, error: 'missing_id' }, cfg.corsOrigin); - return; - } - - try { - const revokedId = await revokeApiToken(cfg, id); - sendJson(res, 200, { ok: true, id: revokedId }, cfg.corsOrigin); - } catch (err) { - sendJson(res, 500, { ok: false, error: String(err?.message || err) }, cfg.corsOrigin); - } - return; - } - - sendJson(res, 404, { ok: false, error: 'not_found' }, cfg.corsOrigin); -} - -function main() { - const cfg = resolveConfig(); - const server = http.createServer((req, res) => void handler(cfg, req, res)); - - server.listen(cfg.port, () => { - console.log( - JSON.stringify( - { - service: 'trade-api', - version: cfg.appVersion, - buildTimestamp: cfg.buildTimestamp, - port: cfg.port, - hasuraUrl: cfg.hasuraUrl, - ticksTable: cfg.ticksTable, - candlesFunction: cfg.candlesFunction, - hasuraAdminSecret: cfg.hasuraAdminSecret ? '***' : undefined, - apiAdminSecret: cfg.apiAdminSecret ? '***' : undefined, - }, - null, - 2 - ) - ); - }); -} - -main(); diff --git a/kustomize/base/api/service.yaml b/kustomize/base/api/service.yaml deleted file mode 100644 index 43e0d4e..0000000 --- a/kustomize/base/api/service.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: trade-api -spec: - type: ClusterIP - selector: - app.kubernetes.io/name: trade-api - ports: - - name: http - port: 8787 - targetPort: http diff --git a/kustomize/base/api/wrapper.mjs b/kustomize/base/api/wrapper.mjs deleted file mode 100644 index 197ead9..0000000 --- a/kustomize/base/api/wrapper.mjs +++ /dev/null @@ -1,783 +0,0 @@ -import crypto from 'node:crypto'; -import http from 'node:http'; -import { spawn } from 'node:child_process'; - -const WRAPPER_PORT = Number.parseInt(String(process.env.PORT || '8787'), 10); -const UPSTREAM_PORT = Number.parseInt(String(process.env.UPSTREAM_PORT || '8788'), 10); -const UPSTREAM_HOST = String(process.env.UPSTREAM_HOST || '127.0.0.1'); -const UPSTREAM_ENTRY = String(process.env.UPSTREAM_ENTRY || '/app/services/api/server.mjs'); - -const HASURA_URL = String(process.env.HASURA_GRAPHQL_URL || 'http://hasura:8080/v1/graphql'); -const HASURA_ADMIN_SECRET = String(process.env.HASURA_ADMIN_SECRET || ''); -const CORS_ORIGIN = String(process.env.CORS_ORIGIN || '*'); - -if (!Number.isInteger(WRAPPER_PORT) || WRAPPER_PORT <= 0) throw new Error('Invalid PORT'); -if (!Number.isInteger(UPSTREAM_PORT) || UPSTREAM_PORT <= 0) throw new Error('Invalid UPSTREAM_PORT'); - -function getIsoNow() { - return new Date().toISOString(); -} - -function sha256Hex(text) { - return crypto.createHash('sha256').update(text, 'utf8').digest('hex'); -} - -function getHeader(req, name) { - const v = req.headers[String(name).toLowerCase()]; - return Array.isArray(v) ? v[0] : v; -} - -function readBearerToken(req) { - const auth = getHeader(req, 'authorization'); - if (auth && typeof auth === 'string') { - const m = auth.match(/^Bearer\s+(.+)$/i); - if (m && m[1]) return m[1].trim(); - } - const apiKey = getHeader(req, 'x-api-key'); - if (apiKey && typeof apiKey === 'string' && apiKey.trim()) return apiKey.trim(); - return undefined; -} - -function withCors(res) { - res.setHeader('access-control-allow-origin', CORS_ORIGIN); - res.setHeader('access-control-allow-methods', 'GET,POST,OPTIONS'); - res.setHeader( - 'access-control-allow-headers', - 'content-type, authorization, x-api-key, x-admin-secret' - ); -} - -function sendJson(res, status, body) { - withCors(res); - res.statusCode = status; - res.setHeader('content-type', 'application/json; charset=utf-8'); - res.end(JSON.stringify(body)); -} - -async function readBodyJson(req, { maxBytes }) { - const chunks = []; - let total = 0; - for await (const chunk of req) { - total += chunk.length; - if (total > maxBytes) throw new Error('payload_too_large'); - chunks.push(chunk); - } - const text = Buffer.concat(chunks).toString('utf8'); - if (!text.trim()) return {}; - try { - return JSON.parse(text); - } catch { - throw new Error('invalid_json'); - } -} - -function parseNumeric(value) { - if (value == null) return null; - if (typeof value === 'number') return Number.isFinite(value) ? value : null; - if (typeof value === 'string') { - const s = value.trim(); - if (!s) return null; - const n = Number(s); - return Number.isFinite(n) ? n : null; - } - return null; -} - -function clampInt(value, min, max) { - const n = Number.parseInt(String(value), 10); - if (!Number.isFinite(n) || !Number.isInteger(n)) return min; - return Math.min(max, Math.max(min, n)); -} - -function clampNumber(value, min, max, fallback) { - const n = typeof value === 'number' ? value : Number(value); - if (!Number.isFinite(n)) return fallback; - return Math.min(max, Math.max(min, n)); -} - -function isUuid(value) { - const s = String(value || '').trim(); - return /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(s); -} - -async function hasuraRequest(query, variables) { - if (!HASURA_ADMIN_SECRET) throw new Error('Missing HASURA_ADMIN_SECRET'); - const res = await fetch(HASURA_URL, { - method: 'POST', - headers: { - 'content-type': 'application/json', - 'x-hasura-admin-secret': HASURA_ADMIN_SECRET, - }, - body: JSON.stringify({ query, variables }), - }); - const text = await res.text(); - if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`); - const json = text ? JSON.parse(text) : {}; - if (json.errors?.length) throw new Error(json.errors.map((e) => e.message).join(' | ')); - return json.data; -} - -function normalizeScopes(value) { - if (!value) return []; - if (Array.isArray(value)) return value.map((v) => String(v)).filter(Boolean); - if (typeof value === 'string') { - return value - .split(',') - .map((s) => s.trim()) - .filter(Boolean); - } - return []; -} - -async function requireValidToken(req, requiredScope) { - const token = readBearerToken(req); - if (!token) return { ok: false, status: 401, error: 'missing_token' }; - - const hash = sha256Hex(token); - const query = ` - query ValidToken($hash: String!) { - api_tokens(where: {token_hash: {_eq: $hash}, revoked_at: {_is_null: true}}, limit: 1) { - id - name - scopes - } - } - `; - - let data; - try { - data = await hasuraRequest(query, { hash }); - } catch (err) { - return { ok: false, status: 500, error: String(err?.message || err) }; - } - - const row = data?.api_tokens?.[0]; - if (!row?.id) return { ok: false, status: 401, error: 'invalid_or_revoked_token' }; - const scopes = normalizeScopes(row.scopes); - if (requiredScope && !scopes.includes(requiredScope)) { - return { ok: false, status: 403, error: 'missing_scope' }; - } - - // best-effort touch - const touch = ` - mutation TouchToken($id: uuid!, $ts: timestamptz!) { - update_api_tokens_by_pk(pk_columns: {id: $id}, _set: {last_used_at: $ts}) { id } - } - `; - hasuraRequest(touch, { id: row.id, ts: getIsoNow() }).catch(() => {}); - - return { ok: true, token: { id: row.id, name: row.name } }; -} - -async function readSolPriceUsd() { - try { - const q = ` - query SolPrice { - dlob_stats_latest(where: {market_name: {_eq: "SOL-PERP"}}, limit: 1) { - mid_price - mark_price - oracle_price - } - } - `; - const data = await hasuraRequest(q, {}); - const row = data?.dlob_stats_latest?.[0]; - const p = parseNumeric(row?.mid_price) ?? parseNumeric(row?.mark_price) ?? parseNumeric(row?.oracle_price); - if (p != null && p > 0) return p; - } catch { - // ignore - } - return null; -} - -function getByPath(obj, pathStr) { - if (!obj || typeof obj !== 'object') return undefined; - const parts = String(pathStr || '').split('.').filter(Boolean); - let cur = obj; - for (const p of parts) { - if (!cur || typeof cur !== 'object') return undefined; - cur = cur[p]; - } - return cur; -} - -function readNumberFromPayload(payload, paths) { - for (const p of paths) { - const v = getByPath(payload, p); - const n = parseNumeric(v); - if (n != null) return n; - } - return null; -} - -function readTextFromPayload(payload, paths) { - for (const p of paths) { - const v = getByPath(payload, p); - if (typeof v === 'string' && v.trim()) return v.trim(); - } - return null; -} - -function inferContractSizeUsd(contract) { - return ( - readNumberFromPayload(contract, [ - 'desired.size_usd', - 'desired.sizeUsd', - 'desired.notional_usd', - 'desired.notionalUsd', - 'entry.size_usd', - 'entry.sizeUsd', - 'entry.notional_usd', - 'entry.notionalUsd', - 'entry.order_intent.size_usd', - 'entry.order_intent.sizeUsd', - 'desired.order_intent.size_usd', - 'desired.order_intent.sizeUsd', - ]) || null - ); -} - -function inferContractSide(contract) { - const raw = - readTextFromPayload(contract, [ - 'desired.side', - 'entry.side', - 'entry.order_intent.side', - 'desired.order_intent.side', - 'desired.direction', - 'entry.direction', - ]) || ''; - const v = raw.toLowerCase(); - if (v === 'long' || v === 'buy') return 'long'; - if (v === 'short' || v === 'sell') return 'short'; - return null; -} - -function sumCostsFromEvents(events) { - const totals = { - tradeFeeUsd: 0, - txFeeUsd: 0, - slippageUsd: 0, - fundingUsd: 0, - realizedPnlUsd: 0, - txCount: 0, - fillCount: 0, - cancelCount: 0, - modifyCount: 0, - errorCount: 0, - }; - - for (const ev of events || []) { - const t = String(ev?.event_type || '').toLowerCase(); - const payload = ev?.payload && typeof ev.payload === 'object' ? ev.payload : {}; - - const tradeFeeUsd = - readNumberFromPayload(payload, [ - 'realized_fee_usd', - 'trade_fee_usd', - 'fee_usd', - 'fees.trade_fee_usd', - 'fees.usd', - ]) || 0; - const txFeeUsd = - readNumberFromPayload(payload, [ - 'realized_tx_usd', - 'tx_fee_usd', - 'network_fee_usd', - 'fees.tx_fee_usd', - 'fees.network_usd', - ]) || 0; - const slippageUsd = - readNumberFromPayload(payload, [ - 'slippage_usd', - 'realized_slippage_usd', - 'execution_usd', - 'realized_execution_usd', - ]) || 0; - const fundingUsd = readNumberFromPayload(payload, ['funding_usd', 'realized_funding_usd']) || 0; - const pnlUsd = readNumberFromPayload(payload, ['realized_pnl_usd', 'pnl_usd']) || 0; - const txCount = readNumberFromPayload(payload, ['tx_count', 'txCount']) || 0; - - totals.tradeFeeUsd += tradeFeeUsd; - totals.txFeeUsd += txFeeUsd; - totals.slippageUsd += slippageUsd; - totals.fundingUsd += fundingUsd; - totals.realizedPnlUsd += pnlUsd; - totals.txCount += txCount; - - if (t.includes('fill')) totals.fillCount += 1; - if (t.includes('cancel')) totals.cancelCount += 1; - if (t.includes('modify') || t.includes('reprice')) totals.modifyCount += 1; - if (t.includes('error') || String(ev?.severity || '').toLowerCase() === 'error') totals.errorCount += 1; - } - - const totalCostsUsd = totals.tradeFeeUsd + totals.txFeeUsd + totals.slippageUsd + totals.fundingUsd; - - return { - ...totals, - totalCostsUsd, - netPnlUsd: totals.realizedPnlUsd - totalCostsUsd, - }; -} - -function buildCostSeriesFromEvents(events, { maxPoints }) { - const points = []; - const totals = { - tradeFeeUsd: 0, - txFeeUsd: 0, - slippageUsd: 0, - fundingUsd: 0, - realizedPnlUsd: 0, - }; - - for (const ev of events || []) { - const ts = ev?.ts; - if (!ts) continue; - const payload = ev?.payload && typeof ev.payload === 'object' ? ev.payload : {}; - - const tradeFeeUsd = - readNumberFromPayload(payload, [ - 'realized_fee_usd', - 'trade_fee_usd', - 'fee_usd', - 'fees.trade_fee_usd', - 'fees.usd', - ]) || 0; - const txFeeUsd = - readNumberFromPayload(payload, [ - 'realized_tx_usd', - 'tx_fee_usd', - 'network_fee_usd', - 'fees.tx_fee_usd', - 'fees.network_usd', - ]) || 0; - const slippageUsd = - readNumberFromPayload(payload, [ - 'slippage_usd', - 'realized_slippage_usd', - 'execution_usd', - 'realized_execution_usd', - ]) || 0; - const fundingUsd = readNumberFromPayload(payload, ['funding_usd', 'realized_funding_usd']) || 0; - const pnlUsd = readNumberFromPayload(payload, ['realized_pnl_usd', 'pnl_usd']) || 0; - - totals.tradeFeeUsd += tradeFeeUsd; - totals.txFeeUsd += txFeeUsd; - totals.slippageUsd += slippageUsd; - totals.fundingUsd += fundingUsd; - totals.realizedPnlUsd += pnlUsd; - - const totalCostsUsd = totals.tradeFeeUsd + totals.txFeeUsd + totals.slippageUsd + totals.fundingUsd; - points.push({ - ts, - tradeFeeUsd: totals.tradeFeeUsd, - txFeeUsd: totals.txFeeUsd, - slippageUsd: totals.slippageUsd, - fundingUsd: totals.fundingUsd, - totalCostsUsd, - realizedPnlUsd: totals.realizedPnlUsd, - netPnlUsd: totals.realizedPnlUsd - totalCostsUsd, - }); - } - - const cap = Math.max(50, Math.min(10_000, Number(maxPoints) || 600)); - if (points.length <= cap) return points; - - const step = Math.ceil(points.length / cap); - const sampled = []; - for (let i = 0; i < points.length; i += step) sampled.push(points[i]); - const last = points[points.length - 1]; - if (sampled[sampled.length - 1] !== last) sampled.push(last); - return sampled; -} - -function proxyToUpstream(req, res) { - const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`); - - const headers = { ...req.headers }; - delete headers.host; - delete headers.connection; - - const opts = { - host: UPSTREAM_HOST, - port: UPSTREAM_PORT, - method: req.method, - path: url.pathname + url.search, - headers, - }; - - const upstreamReq = http.request(opts, (upstreamRes) => { - withCors(res); - res.statusCode = upstreamRes.statusCode || 502; - - for (const [k, v] of Object.entries(upstreamRes.headers || {})) { - if (!k) continue; - if (k.toLowerCase() === 'content-length') continue; - if (k.toLowerCase().startsWith('access-control-')) continue; - if (v != null) res.setHeader(k, v); - } - - upstreamRes.pipe(res); - }); - - upstreamReq.on('error', (err) => { - sendJson(res, 502, { ok: false, error: String(err?.message || err) }); - }); - - req.pipe(upstreamReq); -} - -async function handleMonitor(req, res, url) { - const auth = await requireValidToken(req, 'read'); - if (!auth.ok) { - sendJson(res, auth.status, { ok: false, error: auth.error }); - return; - } - - const pathname = url.pathname; - const parts = pathname.split('/').filter(Boolean); - const contractId = parts[2]; - if (!isUuid(contractId)) { - sendJson(res, 400, { ok: false, error: 'invalid_contract_id' }); - return; - } - - const limit = clampInt(url.searchParams.get('eventsLimit') || '2000', 10, 50_000); - const wantSeries = (url.searchParams.get('series') || '').trim() === '1'; - const seriesMax = clampInt(url.searchParams.get('seriesMax') || '600', 50, 10_000); - - const qContract = ` - query ContractByPk($id: uuid!) { - bot_contracts_by_pk(id: $id) { - id - decision_id - bot_id - model_version - market_name - subaccount_id - status - desired - entry - manage - exit - gates - created_at - updated_at - last_heartbeat_at - ended_at - reason - } - } - `; - const qEvents = ` - query ContractEvents($id: uuid!, $limit: Int!) { - bot_events(where: {contract_id: {_eq: $id}}, order_by: {ts: asc}, limit: $limit) { - ts - contract_id - decision_id - bot_id - market_name - event_type - severity - payload - } - } - `; - - try { - const data = await hasuraRequest(qContract, { id: contractId }); - const contract = data?.bot_contracts_by_pk; - if (!contract?.id) { - sendJson(res, 404, { ok: false, error: 'contract_not_found' }); - return; - } - - const evData = await hasuraRequest(qEvents, { id: contractId, limit }); - const events = evData?.bot_events || []; - const costs = sumCostsFromEvents(events); - const series = wantSeries ? buildCostSeriesFromEvents(events, { maxPoints: seriesMax }) : null; - - const sizeUsd = inferContractSizeUsd(contract); - const side = inferContractSide(contract); - - let closeEst = null; - if (contract.market_name && sizeUsd != null) { - const qSlip = ` - query Slippage($market: String!) { - dlob_slippage_latest(where: {market_name: {_eq: $market}}) { - market_name - side - size_usd - mid_price - vwap_price - worst_price - impact_bps - fill_pct - updated_at - } - } - `; - const slipData = await hasuraRequest(qSlip, { market: contract.market_name }); - const rows = slipData?.dlob_slippage_latest || []; - - const pickNearest = (wantedSide) => { - const candidates = rows.filter((r) => String(r.side || '').toLowerCase() === wantedSide); - if (!candidates.length) return null; - let best = null; - let bestD = Infinity; - for (const r of candidates) { - const s = parseNumeric(r.size_usd); - if (s == null) continue; - const d = Math.abs(s - sizeUsd); - if (d < bestD) { - bestD = d; - best = r; - } - } - return best; - }; - - closeEst = { - requestedSizeUsd: sizeUsd, - entrySide: side, - suggestedCloseSide: side === 'long' ? 'sell' : side === 'short' ? 'buy' : null, - buy: pickNearest('buy'), - sell: pickNearest('sell'), - }; - } - - sendJson(res, 200, { - ok: true, - contract, - eventsCount: events.length, - costs, - series, - closeEstimate: closeEst, - }); - } catch (err) { - sendJson(res, 500, { ok: false, error: String(err?.message || err) }); - } -} - -async function handleEstimate(req, res) { - const auth = await requireValidToken(req, 'read'); - if (!auth.ok) { - sendJson(res, auth.status, { ok: false, error: auth.error }); - return; - } - - let body; - try { - body = await readBodyJson(req, { maxBytes: 256 * 1024 }); - } catch (err) { - const msg = String(err?.message || err); - if (msg === 'payload_too_large') { - sendJson(res, 413, { ok: false, error: 'payload_too_large' }); - return; - } - sendJson(res, 400, { ok: false, error: 'invalid_json' }); - return; - } - - const market = String(body?.market_name || body?.market || '').trim(); - if (!market) { - sendJson(res, 400, { ok: false, error: 'missing_market' }); - return; - } - - const notionalUsd = parseNumeric(body?.notional_usd ?? body?.notionalUsd ?? body?.size_usd ?? body?.sizeUsd); - if (!(notionalUsd != null && notionalUsd > 0)) { - sendJson(res, 400, { ok: false, error: 'invalid_notional_usd' }); - return; - } - - const entrySideRaw = String(body?.side || 'long').trim().toLowerCase(); - const entrySide = entrySideRaw === 'short' ? 'short' : 'long'; - - const orderType = String(body?.order_type || body?.orderType || 'market').trim().toLowerCase(); - const isMarket = orderType === 'market' || orderType === 'taker'; - - const takerBps = clampNumber(parseNumeric(body?.fee_taker_bps) ?? 5, 0, 1000, 5); - const makerBps = clampNumber(parseNumeric(body?.fee_maker_bps) ?? 0, -1000, 1000, 0); - const feeBps = isMarket ? takerBps : makerBps; - - let txFeeUsdEst = parseNumeric(body?.tx_fee_usd_est); - if (txFeeUsdEst == null) { - const baseLamports = 5000; - const sigs = 1; - const priorityLamports = 0; - const lamports = baseLamports * sigs + priorityLamports; - const sol = lamports / 1_000_000_000; - const solUsd = await readSolPriceUsd(); - txFeeUsdEst = solUsd != null ? sol * solUsd : 0; - } - txFeeUsdEst = clampNumber(txFeeUsdEst, 0, 100, 0); - - const expectedReprices = clampInt(body?.expected_reprices_per_entry ?? body?.expectedReprices ?? '0', 0, 500); - const modifyTxCount = clampInt(body?.modify_tx_count ?? body?.modifyTxCount ?? '2', 0, 10); - - try { - const qSlip = ` - query Slippage($market: String!) { - dlob_slippage_latest(where: {market_name: {_eq: $market}}) { - market_name - side - size_usd - mid_price - vwap_price - worst_price - impact_bps - fill_pct - updated_at - } - } - `; - const slipData = await hasuraRequest(qSlip, { market }); - const rows = slipData?.dlob_slippage_latest || []; - const wantedSide = entrySide === 'long' ? 'buy' : 'sell'; - const candidates = rows.filter((r) => String(r.side || '').toLowerCase() === wantedSide); - - let best = null; - let bestD = Infinity; - for (const r of candidates) { - const s = parseNumeric(r.size_usd); - if (s == null) continue; - const d = Math.abs(s - notionalUsd); - if (d < bestD) { - bestD = d; - best = r; - } - } - - const impactBps = parseNumeric(best?.impact_bps) ?? 0; - const slippageUsd = (notionalUsd * impactBps) / 10_000; - const tradeFeeUsd = (notionalUsd * feeBps) / 10_000; - const modifyCostUsd = expectedReprices * modifyTxCount * txFeeUsdEst; - const totalUsd = tradeFeeUsd + slippageUsd + txFeeUsdEst + modifyCostUsd; - const totalBps = (totalUsd / notionalUsd) * 10_000; - - sendJson(res, 200, { - ok: true, - input: { - market_name: market, - notional_usd: notionalUsd, - side: entrySide, - order_type: orderType, - fee_bps: feeBps, - tx_fee_usd_est: txFeeUsdEst, - expected_reprices_per_entry: expectedReprices, - }, - dlob: best - ? { - size_usd: best.size_usd, - side: best.side, - mid_price: best.mid_price, - vwap_price: best.vwap_price, - impact_bps: best.impact_bps, - fill_pct: best.fill_pct, - updated_at: best.updated_at, - } - : null, - breakdown: { - trade_fee_usd: tradeFeeUsd, - slippage_usd: slippageUsd, - tx_fee_usd: txFeeUsdEst, - expected_modify_usd: modifyCostUsd, - total_usd: totalUsd, - total_bps: totalBps, - breakeven_bps: totalBps, - }, - }); - } catch (err) { - sendJson(res, 500, { ok: false, error: String(err?.message || err) }); - } -} - -let upstreamChild = null; - -function startUpstream() { - const env = { ...process.env, PORT: String(UPSTREAM_PORT) }; - upstreamChild = spawn('node', [UPSTREAM_ENTRY], { env, stdio: 'inherit' }); - upstreamChild.on('exit', (code, signal) => { - console.error(`upstream exited: code=${code} signal=${signal}`); - }); -} - -function shutdown() { - if (upstreamChild && !upstreamChild.killed) upstreamChild.kill('SIGTERM'); -} - -process.on('SIGTERM', shutdown); -process.on('SIGINT', shutdown); - -startUpstream(); - -const server = http.createServer(async (req, res) => { - if (req.method === 'OPTIONS') { - withCors(res); - res.statusCode = 204; - res.end(); - return; - } - - const url = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`); - const pathname = url.pathname; - - if (req.method === 'GET' && pathname === '/healthz') { - // Check upstream quickly; if it's down, we fail readiness. - const opts = { host: UPSTREAM_HOST, port: UPSTREAM_PORT, path: '/healthz', method: 'GET', timeout: 800 }; - const upstreamOk = await new Promise((resolve) => { - const r = http.request(opts, (rr) => { - rr.resume(); - resolve(rr.statusCode === 200); - }); - r.on('timeout', () => { - r.destroy(); - resolve(false); - }); - r.on('error', () => resolve(false)); - r.end(); - }); - - if (!upstreamOk) { - sendJson(res, 503, { ok: false, error: 'upstream_not_ready' }); - return; - } - - sendJson(res, 200, { - ok: true, - service: 'trade-api-wrapper', - startedAt: getIsoNow(), - upstream: { host: UPSTREAM_HOST, port: UPSTREAM_PORT }, - }); - return; - } - - if (req.method === 'POST' && pathname === '/v1/contracts/costs/estimate') { - await handleEstimate(req, res); - return; - } - - if (req.method === 'GET' && pathname.startsWith('/v1/contracts/') && pathname.endsWith('/monitor')) { - await handleMonitor(req, res, url); - return; - } - - proxyToUpstream(req, res); -}); - -server.listen(WRAPPER_PORT, () => { - console.log( - JSON.stringify( - { - service: 'trade-api-wrapper', - port: WRAPPER_PORT, - upstream: { entry: UPSTREAM_ENTRY, host: UPSTREAM_HOST, port: UPSTREAM_PORT }, - hasuraUrl: HASURA_URL, - hasuraAdminSecret: HASURA_ADMIN_SECRET ? '***' : undefined, - }, - null, - 2 - ) - ); -}); diff --git a/kustomize/base/dlob-depth-worker/deployment-drift.yaml b/kustomize/base/dlob-depth-worker/deployment-drift.yaml deleted file mode 100644 index 99ae85a..0000000 --- a/kustomize/base/dlob-depth-worker/deployment-drift.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dlob-depth-worker-drift - annotations: - argocd.argoproj.io/sync-wave: "6" -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: dlob-depth-worker-drift - template: - metadata: - labels: - app.kubernetes.io/name: dlob-depth-worker-drift - spec: - containers: - - name: worker - image: node:20-slim - imagePullPolicy: IfNotPresent - env: - - name: HASURA_GRAPHQL_URL - value: http://hasura:8080/v1/graphql - - name: HASURA_ADMIN_SECRET - valueFrom: - secretKeyRef: - name: trade-hasura - key: HASURA_GRAPHQL_ADMIN_SECRET - - name: DLOB_SOURCE - value: drift - - name: DLOB_MARKETS - value: SOL-PERP,DOGE-PERP,JUP-PERP - - name: DLOB_POLL_MS - value: "1000" - - name: DLOB_DEPTH_BPS_BANDS - value: "5,10,20,50,100,200" - - name: PRICE_PRECISION - value: "1000000" - - name: BASE_PRECISION - value: "1000000000" - command: ["node", "/app/worker.mjs"] - volumeMounts: - - name: script - mountPath: /app/worker.mjs - subPath: worker.mjs - readOnly: true - volumes: - - name: script - configMap: - name: dlob-depth-worker-script diff --git a/kustomize/base/dlob-depth-worker/deployment.yaml b/kustomize/base/dlob-depth-worker/deployment.yaml deleted file mode 100644 index 1436901..0000000 --- a/kustomize/base/dlob-depth-worker/deployment.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dlob-depth-worker - annotations: - argocd.argoproj.io/sync-wave: "6" -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: dlob-depth-worker - template: - metadata: - labels: - app.kubernetes.io/name: dlob-depth-worker - spec: - containers: - - name: worker - image: node:20-slim - imagePullPolicy: IfNotPresent - env: - - name: HASURA_GRAPHQL_URL - value: http://hasura:8080/v1/graphql - - name: HASURA_ADMIN_SECRET - valueFrom: - secretKeyRef: - name: trade-hasura - key: HASURA_GRAPHQL_ADMIN_SECRET - - name: DLOB_SOURCE - value: mevnode - - name: DLOB_MARKETS - value: SOL-PERP,DOGE-PERP,JUP-PERP - - name: DLOB_POLL_MS - value: "1000" - - name: DLOB_DEPTH_BPS_BANDS - value: "5,10,20,50,100,200" - - name: PRICE_PRECISION - value: "1000000" - - name: BASE_PRECISION - value: "1000000000" - command: ["node", "/app/worker.mjs"] - volumeMounts: - - name: script - mountPath: /app/worker.mjs - subPath: worker.mjs - readOnly: true - volumes: - - name: script - configMap: - name: dlob-depth-worker-script diff --git a/kustomize/base/dlob-depth-worker/worker.mjs b/kustomize/base/dlob-depth-worker/worker.mjs deleted file mode 100644 index 85b0a56..0000000 --- a/kustomize/base/dlob-depth-worker/worker.mjs +++ /dev/null @@ -1,316 +0,0 @@ -import process from 'node:process'; -import { setTimeout as sleep } from 'node:timers/promises'; - -function getIsoNow() { - return new Date().toISOString(); -} - -function clampInt(value, min, max, fallback) { - const n = Number.parseInt(String(value ?? ''), 10); - if (!Number.isInteger(n)) return fallback; - return Math.min(max, Math.max(min, n)); -} - -function envList(name, fallbackCsv) { - const raw = process.env[name] ?? fallbackCsv; - return String(raw) - .split(',') - .map((s) => s.trim()) - .filter(Boolean); -} - -function envIntList(name, fallbackCsv) { - const out = []; - for (const item of envList(name, fallbackCsv)) { - const n = Number.parseInt(item, 10); - if (!Number.isFinite(n)) continue; - out.push(n); - } - return out.length ? out : envList(name, fallbackCsv).map((v) => Number.parseInt(v, 10)).filter(Number.isFinite); -} - -function toNumberOrNull(value) { - if (value == null) return null; - if (typeof value === 'number') return Number.isFinite(value) ? value : null; - if (typeof value === 'string') { - const s = value.trim(); - if (!s) return null; - const n = Number(s); - return Number.isFinite(n) ? n : null; - } - return null; -} - -function numStr(value) { - if (value == null) return null; - if (typeof value === 'number') return Number.isFinite(value) ? String(value) : null; - if (typeof value === 'string') return value.trim() || null; - return null; -} - -function jsonNormalize(value) { - if (typeof value !== 'string') return value; - const s = value.trim(); - if (!s) return null; - try { - return JSON.parse(s); - } catch { - return value; - } -} - -function resolveConfig() { - const hasuraUrl = process.env.HASURA_GRAPHQL_URL || 'http://hasura:8080/v1/graphql'; - const hasuraAdminSecret = process.env.HASURA_ADMIN_SECRET || process.env.HASURA_GRAPHQL_ADMIN_SECRET || undefined; - const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined; - - const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode'; - const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP'); - const pollMs = clampInt(process.env.DLOB_POLL_MS, 250, 60_000, 1000); - const bandsBps = envIntList('DLOB_DEPTH_BPS_BANDS', '5,10,20,50,100,200'); - - const pricePrecision = Number(process.env.PRICE_PRECISION || 1_000_000); - const basePrecision = Number(process.env.BASE_PRECISION || 1_000_000_000); - if (!Number.isFinite(pricePrecision) || pricePrecision <= 0) - throw new Error(`Invalid PRICE_PRECISION: ${process.env.PRICE_PRECISION}`); - if (!Number.isFinite(basePrecision) || basePrecision <= 0) - throw new Error(`Invalid BASE_PRECISION: ${process.env.BASE_PRECISION}`); - - return { - hasuraUrl, - hasuraAdminSecret, - hasuraAuthToken, - dlobSource, - markets, - pollMs, - bandsBps, - pricePrecision, - basePrecision, - }; -} - -async function graphqlRequest(cfg, query, variables) { - const headers = { 'content-type': 'application/json' }; - if (cfg.hasuraAuthToken) { - headers.authorization = `Bearer ${cfg.hasuraAuthToken}`; - } else if (cfg.hasuraAdminSecret) { - headers['x-hasura-admin-secret'] = cfg.hasuraAdminSecret; - } else { - throw new Error('Missing Hasura auth (set HASURA_AUTH_TOKEN or HASURA_ADMIN_SECRET)'); - } - - const res = await fetch(cfg.hasuraUrl, { - method: 'POST', - headers, - body: JSON.stringify({ query, variables }), - signal: AbortSignal.timeout(10_000), - }); - const text = await res.text(); - if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`); - const json = JSON.parse(text); - if (json.errors?.length) throw new Error(json.errors.map((e) => e.message).join(' | ')); - return json.data; -} - -function parseLevels(raw, pricePrecision, basePrecision, side) { - const v = jsonNormalize(raw); - if (!Array.isArray(v)) return []; - - const out = []; - for (const item of v) { - const priceInt = toNumberOrNull(item?.price); - const sizeInt = toNumberOrNull(item?.size); - if (priceInt == null || sizeInt == null) continue; - const price = priceInt / pricePrecision; - const size = sizeInt / basePrecision; - if (!Number.isFinite(price) || !Number.isFinite(size)) continue; - out.push({ price, size }); - } - - if (side === 'bid') out.sort((a, b) => b.price - a.price); - if (side === 'ask') out.sort((a, b) => a.price - b.price); - return out; -} - -function computeMid(bestBid, bestAsk, markPrice, oraclePrice) { - if (bestBid != null && bestAsk != null) return (bestBid + bestAsk) / 2; - if (markPrice != null) return markPrice; - if (oraclePrice != null) return oraclePrice; - return null; -} - -function computeBandDepth({ bids, asks, mid, bandBps }) { - if (mid == null || !(mid > 0)) { - return { bidBase: 0, askBase: 0, bidUsd: 0, askUsd: 0, imbalance: null }; - } - - const minBidPrice = mid * (1 - bandBps / 10_000); - const maxAskPrice = mid * (1 + bandBps / 10_000); - - let bidBase = 0; - let askBase = 0; - let bidUsd = 0; - let askUsd = 0; - - for (const lvl of bids) { - if (lvl.price < minBidPrice) break; - bidBase += lvl.size; - bidUsd += lvl.size * lvl.price; - } - - for (const lvl of asks) { - if (lvl.price > maxAskPrice) break; - askBase += lvl.size; - askUsd += lvl.size * lvl.price; - } - - const denom = bidUsd + askUsd; - const imbalance = denom > 0 ? (bidUsd - askUsd) / denom : null; - return { bidBase, askBase, bidUsd, askUsd, imbalance }; -} - -async function fetchL2Latest(cfg) { - const query = ` - query DlobL2Latest($source: String!, $markets: [String!]!) { - dlob_l2_latest(where: {source: {_eq: $source}, market_name: {_in: $markets}}) { - source - market_name - market_type - market_index - ts - slot - mark_price - oracle_price - best_bid_price - best_ask_price - bids - asks - updated_at - } - } - `; - const data = await graphqlRequest(cfg, query, { source: cfg.dlobSource, markets: cfg.markets }); - return Array.isArray(data?.dlob_l2_latest) ? data.dlob_l2_latest : []; -} - -async function upsertDepth(cfg, rows) { - if (!rows.length) return; - const mutation = ` - mutation UpsertDlobDepth($rows: [dlob_depth_bps_latest_insert_input!]!) { - insert_dlob_depth_bps_latest( - objects: $rows - on_conflict: { - constraint: dlob_depth_bps_latest_pkey - update_columns: [ - market_type - market_index - ts - slot - mid_price - best_bid_price - best_ask_price - bid_base - ask_base - bid_usd - ask_usd - imbalance - raw - updated_at - ] - } - ) { affected_rows } - } - `; - await graphqlRequest(cfg, mutation, { rows }); -} - -async function main() { - const cfg = resolveConfig(); - const lastUpdatedAtByMarket = new Map(); - - console.log( - JSON.stringify( - { - service: 'dlob-depth-worker', - startedAt: getIsoNow(), - hasuraUrl: cfg.hasuraUrl, - hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none', - dlobSource: cfg.dlobSource, - markets: cfg.markets, - pollMs: cfg.pollMs, - bandsBps: cfg.bandsBps, - pricePrecision: cfg.pricePrecision, - basePrecision: cfg.basePrecision, - }, - null, - 2 - ) - ); - - while (true) { - const rows = []; - - try { - const l2Rows = await fetchL2Latest(cfg); - for (const l2 of l2Rows) { - const market = String(l2.market_name || '').trim(); - if (!market) continue; - - const updatedAt = l2.updated_at || null; - if (updatedAt && lastUpdatedAtByMarket.get(market) === updatedAt) continue; - if (updatedAt) lastUpdatedAtByMarket.set(market, updatedAt); - - const bestBid = toNumberOrNull(l2.best_bid_price); - const bestAsk = toNumberOrNull(l2.best_ask_price); - const markPrice = toNumberOrNull(l2.mark_price); - const oraclePrice = toNumberOrNull(l2.oracle_price); - const mid = computeMid(bestBid, bestAsk, markPrice, oraclePrice); - - const bids = parseLevels(l2.bids, cfg.pricePrecision, cfg.basePrecision, 'bid'); - const asks = parseLevels(l2.asks, cfg.pricePrecision, cfg.basePrecision, 'ask'); - - for (const bandBps of cfg.bandsBps) { - const d = computeBandDepth({ bids, asks, mid, bandBps }); - rows.push({ - source: cfg.dlobSource, - market_name: market, - band_bps: bandBps, - market_type: l2.market_type ? String(l2.market_type) : 'perp', - market_index: typeof l2.market_index === 'number' ? l2.market_index : null, - ts: l2.ts == null ? null : String(l2.ts), - slot: l2.slot == null ? null : String(l2.slot), - mid_price: numStr(mid), - best_bid_price: numStr(bestBid), - best_ask_price: numStr(bestAsk), - bid_base: numStr(d.bidBase), - ask_base: numStr(d.askBase), - bid_usd: numStr(d.bidUsd), - ask_usd: numStr(d.askUsd), - imbalance: numStr(d.imbalance), - raw: { - ref: 'mid', - pricePrecision: cfg.pricePrecision, - basePrecision: cfg.basePrecision, - }, - updated_at: updatedAt, - }); - } - } - } catch (err) { - console.error(`[dlob-depth-worker] fetch/compute: ${String(err?.message || err)}`); - } - - try { - await upsertDepth(cfg, rows); - } catch (err) { - console.error(`[dlob-depth-worker] upsert: ${String(err?.message || err)}`); - } - - await sleep(cfg.pollMs); - } -} - -main().catch((err) => { - console.error(String(err?.stack || err)); - process.exitCode = 1; -}); diff --git a/kustomize/base/dlob-slippage-worker/deployment-drift.yaml b/kustomize/base/dlob-slippage-worker/deployment-drift.yaml deleted file mode 100644 index 0db2703..0000000 --- a/kustomize/base/dlob-slippage-worker/deployment-drift.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dlob-slippage-worker-drift - annotations: - argocd.argoproj.io/sync-wave: "6" -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: dlob-slippage-worker-drift - template: - metadata: - labels: - app.kubernetes.io/name: dlob-slippage-worker-drift - spec: - containers: - - name: worker - image: node:20-slim - imagePullPolicy: IfNotPresent - env: - - name: HASURA_GRAPHQL_URL - value: http://hasura:8080/v1/graphql - - name: HASURA_ADMIN_SECRET - valueFrom: - secretKeyRef: - name: trade-hasura - key: HASURA_GRAPHQL_ADMIN_SECRET - - name: DLOB_SOURCE - value: drift - - name: DLOB_MARKETS - value: SOL-PERP,DOGE-PERP,JUP-PERP - - name: DLOB_POLL_MS - value: "1000" - - name: DLOB_SLIPPAGE_SIZES_USD - value: "0.1,0.2,0.5,1,2,5,10,25,50,100,250,500,1000,5000,10000,50000" - - name: PRICE_PRECISION - value: "1000000" - - name: BASE_PRECISION - value: "1000000000" - command: ["node", "/app/worker.mjs"] - volumeMounts: - - name: script - mountPath: /app/worker.mjs - subPath: worker.mjs - readOnly: true - volumes: - - name: script - configMap: - name: dlob-slippage-worker-script diff --git a/kustomize/base/dlob-slippage-worker/deployment.yaml b/kustomize/base/dlob-slippage-worker/deployment.yaml deleted file mode 100644 index f2915cd..0000000 --- a/kustomize/base/dlob-slippage-worker/deployment.yaml +++ /dev/null @@ -1,50 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dlob-slippage-worker - annotations: - argocd.argoproj.io/sync-wave: "6" -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: dlob-slippage-worker - template: - metadata: - labels: - app.kubernetes.io/name: dlob-slippage-worker - spec: - containers: - - name: worker - image: node:20-slim - imagePullPolicy: IfNotPresent - env: - - name: HASURA_GRAPHQL_URL - value: http://hasura:8080/v1/graphql - - name: HASURA_ADMIN_SECRET - valueFrom: - secretKeyRef: - name: trade-hasura - key: HASURA_GRAPHQL_ADMIN_SECRET - - name: DLOB_SOURCE - value: mevnode - - name: DLOB_MARKETS - value: SOL-PERP,DOGE-PERP,JUP-PERP - - name: DLOB_POLL_MS - value: "1000" - - name: DLOB_SLIPPAGE_SIZES_USD - value: "0.1,0.2,0.5,1,2,5,10,25,50,100,250,500,1000,5000,10000,50000" - - name: PRICE_PRECISION - value: "1000000" - - name: BASE_PRECISION - value: "1000000000" - command: ["node", "/app/worker.mjs"] - volumeMounts: - - name: script - mountPath: /app/worker.mjs - subPath: worker.mjs - readOnly: true - volumes: - - name: script - configMap: - name: dlob-slippage-worker-script diff --git a/kustomize/base/dlob-slippage-worker/worker.mjs b/kustomize/base/dlob-slippage-worker/worker.mjs deleted file mode 100644 index c496ec1..0000000 --- a/kustomize/base/dlob-slippage-worker/worker.mjs +++ /dev/null @@ -1,404 +0,0 @@ -import fs from 'node:fs'; -import process from 'node:process'; -import { setTimeout as sleep } from 'node:timers/promises'; - -function readJsonFile(filePath) { - try { - const raw = fs.readFileSync(filePath, 'utf8'); - return JSON.parse(raw); - } catch { - return undefined; - } -} - -function getIsoNow() { - return new Date().toISOString(); -} - -function clampInt(value, min, max, fallback) { - const n = Number.parseInt(String(value ?? ''), 10); - if (!Number.isInteger(n)) return fallback; - return Math.min(max, Math.max(min, n)); -} - -function envList(name, fallbackCsv) { - const raw = process.env[name] ?? fallbackCsv; - return String(raw) - .split(',') - .map((s) => s.trim()) - .filter(Boolean); -} - -function parsePositiveNumber(value) { - const n = Number.parseFloat(String(value ?? '').trim()); - if (!Number.isFinite(n) || !(n > 0)) return null; - return n; -} - -function resolveConfig() { - const tokensPath = - process.env.HASURA_TOKENS_FILE || - process.env.TOKENS_FILE || - process.env.HASURA_CONFIG_FILE || - '/app/tokens/hasura.json'; - const tokens = readJsonFile(tokensPath) || {}; - - const hasuraUrl = - process.env.HASURA_GRAPHQL_URL || - tokens.graphqlUrl || - tokens.apiUrl || - 'http://hasura:8080/v1/graphql'; - const hasuraAdminSecret = - process.env.HASURA_ADMIN_SECRET || - process.env.HASURA_GRAPHQL_ADMIN_SECRET || - tokens.adminSecret || - tokens.hasuraAdminSecret; - const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined; - - const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode'; - const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP'); - const pollMs = clampInt(process.env.DLOB_POLL_MS, 250, 60_000, 1000); - - const sizesUsd = envList('DLOB_SLIPPAGE_SIZES_USD', '10,25,50,100,250,500,1000') - .map(parsePositiveNumber) - .filter((n) => n != null) - .map((n) => n) - .filter((n, idx, arr) => arr.findIndex((x) => x === n) === idx) - .sort((a, b) => a - b); - - const sizesUsdInt = sizesUsd.filter((n) => Number.isInteger(n)); - - const depthLevels = clampInt(process.env.DLOB_DEPTH, 1, 50, 25); - const pricePrecision = Number(process.env.PRICE_PRECISION || 1_000_000); - const basePrecision = Number(process.env.BASE_PRECISION || 1_000_000_000); - if (!Number.isFinite(pricePrecision) || pricePrecision <= 0) throw new Error(`Invalid PRICE_PRECISION: ${process.env.PRICE_PRECISION}`); - if (!Number.isFinite(basePrecision) || basePrecision <= 0) throw new Error(`Invalid BASE_PRECISION: ${process.env.BASE_PRECISION}`); - - return { - hasuraUrl, - hasuraAdminSecret, - hasuraAuthToken, - dlobSource, - markets, - pollMs, - sizesUsd, - sizesUsdInt, - depthLevels, - pricePrecision, - basePrecision, - }; -} - -async function graphqlRequest(cfg, query, variables) { - const headers = { 'content-type': 'application/json' }; - if (cfg.hasuraAuthToken) { - headers.authorization = `Bearer ${cfg.hasuraAuthToken}`; - } else if (cfg.hasuraAdminSecret) { - headers['x-hasura-admin-secret'] = cfg.hasuraAdminSecret; - } else { - throw new Error('Missing Hasura auth (set HASURA_AUTH_TOKEN or HASURA_ADMIN_SECRET or mount tokens/hasura.json)'); - } - - const res = await fetch(cfg.hasuraUrl, { - method: 'POST', - headers, - body: JSON.stringify({ query, variables }), - signal: AbortSignal.timeout(15_000), - }); - const text = await res.text(); - if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`); - const json = JSON.parse(text); - if (json.errors?.length) { - throw new Error(json.errors.map((e) => e.message).join(' | ')); - } - return json.data; -} - -function toNumberOrNull(value) { - if (value == null) return null; - if (typeof value === 'number') return Number.isFinite(value) ? value : null; - if (typeof value === 'string') { - const s = value.trim(); - if (!s) return null; - const n = Number(s); - return Number.isFinite(n) ? n : null; - } - return null; -} - -function normalizeLevels(raw) { - if (raw == null) return []; - if (Array.isArray(raw)) return raw; - if (typeof raw === 'string') { - const s = raw.trim(); - if (!s) return []; - try { - const v = JSON.parse(s); - return Array.isArray(v) ? v : []; - } catch { - return []; - } - } - return []; -} - -function parseScaledLevels(raw, pricePrecision, basePrecision) { - const levels = normalizeLevels(raw); - const out = []; - for (const it of levels) { - const priceInt = toNumberOrNull(it?.price); - const sizeInt = toNumberOrNull(it?.size); - if (priceInt == null || sizeInt == null) continue; - const price = priceInt / pricePrecision; - const base = sizeInt / basePrecision; - if (!Number.isFinite(price) || !Number.isFinite(base)) continue; - out.push({ price, base }); - } - return out; -} - -function simulateFill(levels, sizeUsd) { - let remainingUsd = sizeUsd; - let filledUsd = 0; - let filledBase = 0; - let totalQuoteUsd = 0; - let worstPrice = null; - let levelsConsumed = 0; - - for (const l of levels) { - if (remainingUsd <= 0) break; - const levelUsd = l.base * l.price; - if (levelUsd <= 0) continue; - levelsConsumed += 1; - worstPrice = l.price; - - const takeUsd = Math.min(remainingUsd, levelUsd); - const takeBase = takeUsd / l.price; - - remainingUsd -= takeUsd; - filledUsd += takeUsd; - filledBase += takeBase; - totalQuoteUsd += takeUsd; - } - - const vwapPrice = filledBase > 0 ? totalQuoteUsd / filledBase : null; - const fillPct = sizeUsd > 0 ? (filledUsd / sizeUsd) * 100 : null; - - return { - filledUsd, - filledBase, - vwapPrice, - worstPrice, - levelsConsumed, - fillPct, - }; -} - -function impactBps({ side, mid, vwap }) { - if (mid == null || vwap == null || mid <= 0) return null; - if (side === 'buy') return ((vwap / mid) - 1) * 10_000; - if (side === 'sell') return (1 - (vwap / mid)) * 10_000; - return null; -} - -async function main() { - const cfg = resolveConfig(); - - console.log( - JSON.stringify( - { - service: 'dlob-slippage-worker', - startedAt: getIsoNow(), - hasuraUrl: cfg.hasuraUrl, - hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none', - dlobSource: cfg.dlobSource, - markets: cfg.markets, - pollMs: cfg.pollMs, - sizesUsd: cfg.sizesUsd, - depthLevels: cfg.depthLevels, - }, - null, - 2 - ) - ); - - const lastSeenUpdatedAt = new Map(); // market -> updated_at - - while (true) { - const updatedAt = getIsoNow(); - - try { - const query = ` - query DlobL2Latest($source: String!, $markets: [String!]!) { - dlob_l2_latest(where: { source: { _eq: $source }, market_name: { _in: $markets } }) { - source - market_name - market_type - market_index - ts - slot - best_bid_price - best_ask_price - bids - asks - updated_at - } - } - `; - - const data = await graphqlRequest(cfg, query, { source: cfg.dlobSource, markets: cfg.markets }); - const rows = Array.isArray(data?.dlob_l2_latest) ? data.dlob_l2_latest : []; - - const objectsV1 = []; - const objectsV2 = []; - - for (const row of rows) { - const market = String(row?.market_name || '').trim(); - if (!market) continue; - - const rowUpdatedAt = row?.updated_at ?? null; - if (rowUpdatedAt && lastSeenUpdatedAt.get(market) === rowUpdatedAt) continue; - if (rowUpdatedAt) lastSeenUpdatedAt.set(market, rowUpdatedAt); - - const bestBid = toNumberOrNull(row?.best_bid_price); - const bestAsk = toNumberOrNull(row?.best_ask_price); - if (bestBid == null || bestAsk == null) continue; - - const mid = (bestBid + bestAsk) / 2; - if (!Number.isFinite(mid) || mid <= 0) continue; - - const bids = parseScaledLevels(row?.bids, cfg.pricePrecision, cfg.basePrecision) - .slice() - .sort((a, b) => b.price - a.price) - .slice(0, cfg.depthLevels); - const asks = parseScaledLevels(row?.asks, cfg.pricePrecision, cfg.basePrecision) - .slice() - .sort((a, b) => a.price - b.price) - .slice(0, cfg.depthLevels); - - for (const sizeUsd of cfg.sizesUsd) { - // buy consumes asks (worse prices as you go up) - { - const sim = simulateFill(asks, sizeUsd); - const baseObj = { - source: cfg.dlobSource, - market_name: market, - side: 'buy', - market_type: row?.market_type ?? 'perp', - market_index: row?.market_index ?? null, - ts: row?.ts == null ? null : String(row.ts), - slot: row?.slot == null ? null : String(row.slot), - mid_price: String(mid), - vwap_price: sim.vwapPrice == null ? null : String(sim.vwapPrice), - worst_price: sim.worstPrice == null ? null : String(sim.worstPrice), - filled_usd: String(sim.filledUsd), - filled_base: String(sim.filledBase), - impact_bps: impactBps({ side: 'buy', mid, vwap: sim.vwapPrice }), - levels_consumed: sim.levelsConsumed, - fill_pct: sim.fillPct == null ? null : String(sim.fillPct), - raw: { depthLevels: cfg.depthLevels }, - updated_at: updatedAt, - }; - objectsV2.push({ ...baseObj, size_usd: String(sizeUsd) }); - if (Number.isInteger(sizeUsd)) objectsV1.push({ ...baseObj, size_usd: Math.trunc(sizeUsd) }); - } - - // sell consumes bids (worse prices as you go down) - { - const sim = simulateFill(bids, sizeUsd); - const baseObj = { - source: cfg.dlobSource, - market_name: market, - side: 'sell', - market_type: row?.market_type ?? 'perp', - market_index: row?.market_index ?? null, - ts: row?.ts == null ? null : String(row.ts), - slot: row?.slot == null ? null : String(row.slot), - mid_price: String(mid), - vwap_price: sim.vwapPrice == null ? null : String(sim.vwapPrice), - worst_price: sim.worstPrice == null ? null : String(sim.worstPrice), - filled_usd: String(sim.filledUsd), - filled_base: String(sim.filledBase), - impact_bps: impactBps({ side: 'sell', mid, vwap: sim.vwapPrice }), - levels_consumed: sim.levelsConsumed, - fill_pct: sim.fillPct == null ? null : String(sim.fillPct), - raw: { depthLevels: cfg.depthLevels }, - updated_at: updatedAt, - }; - objectsV2.push({ ...baseObj, size_usd: String(sizeUsd) }); - if (Number.isInteger(sizeUsd)) objectsV1.push({ ...baseObj, size_usd: Math.trunc(sizeUsd) }); - } - } - } - - if (objectsV1.length) { - const mutation = ` - mutation UpsertSlippageV1($rows: [dlob_slippage_latest_insert_input!]!) { - insert_dlob_slippage_latest( - objects: $rows - on_conflict: { - constraint: dlob_slippage_latest_pkey - update_columns: [ - market_type - market_index - ts - slot - mid_price - vwap_price - worst_price - filled_usd - filled_base - impact_bps - levels_consumed - fill_pct - raw - updated_at - ] - } - ) { affected_rows } - } - `; - await graphqlRequest(cfg, mutation, { rows: objectsV1 }); - } - - if (objectsV2.length) { - const mutation = ` - mutation UpsertSlippageV2($rows: [dlob_slippage_latest_v2_insert_input!]!) { - insert_dlob_slippage_latest_v2( - objects: $rows - on_conflict: { - constraint: dlob_slippage_latest_v2_pkey - update_columns: [ - market_type - market_index - ts - slot - mid_price - vwap_price - worst_price - filled_usd - filled_base - impact_bps - levels_consumed - fill_pct - raw - updated_at - ] - } - ) { affected_rows } - } - `; - await graphqlRequest(cfg, mutation, { rows: objectsV2 }); - } - } catch (err) { - console.error(`[dlob-slippage-worker] ${String(err?.message || err)}`); - } - - await sleep(cfg.pollMs); - } -} - -main().catch((err) => { - console.error(String(err?.stack || err)); - process.exitCode = 1; -}); diff --git a/kustomize/base/dlob-worker/deployment-drift.yaml b/kustomize/base/dlob-worker/deployment-drift.yaml deleted file mode 100644 index 36338bd..0000000 --- a/kustomize/base/dlob-worker/deployment-drift.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dlob-worker-drift - annotations: - argocd.argoproj.io/sync-wave: "5" -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: dlob-worker-drift - template: - metadata: - labels: - app.kubernetes.io/name: dlob-worker-drift - spec: - hostNetwork: true - dnsPolicy: ClusterFirstWithHostNet - containers: - - name: worker - image: node:20-slim - imagePullPolicy: IfNotPresent - env: - - name: HASURA_GRAPHQL_URL - value: http://hasura:8080/v1/graphql - - name: HASURA_ADMIN_SECRET - valueFrom: - secretKeyRef: - name: trade-hasura - key: HASURA_GRAPHQL_ADMIN_SECRET - - name: DLOB_SOURCE - value: drift - - name: DLOB_HTTP_URL - value: https://dlob.drift.trade - - name: DLOB_FORCE_IPV6 - value: "true" - - name: DLOB_MARKETS - value: SOL-PERP,DOGE-PERP,JUP-PERP - - name: DLOB_POLL_MS - value: "500" - - name: DLOB_DEPTH - value: "10" - command: ["node", "/app/worker.mjs"] - volumeMounts: - - name: script - mountPath: /app/worker.mjs - subPath: worker.mjs - readOnly: true - volumes: - - name: script - configMap: - name: dlob-worker-script diff --git a/kustomize/base/dlob-worker/deployment.yaml b/kustomize/base/dlob-worker/deployment.yaml deleted file mode 100644 index 72ed7b9..0000000 --- a/kustomize/base/dlob-worker/deployment.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dlob-worker - annotations: - argocd.argoproj.io/sync-wave: "5" -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: dlob-worker - template: - metadata: - labels: - app.kubernetes.io/name: dlob-worker - spec: - containers: - - name: worker - image: node:20-slim - imagePullPolicy: IfNotPresent - env: - - name: HASURA_GRAPHQL_URL - value: http://hasura:8080/v1/graphql - - name: HASURA_ADMIN_SECRET - valueFrom: - secretKeyRef: - name: trade-hasura - key: HASURA_GRAPHQL_ADMIN_SECRET - - name: DLOB_SOURCE - value: mevnode - - name: DLOB_HTTP_URL - value: http://dlob-server:6969 - - name: DLOB_MARKETS - value: SOL-PERP,DOGE-PERP,JUP-PERP - - name: DLOB_POLL_MS - value: "500" - - name: DLOB_DEPTH - value: "10" - command: ["node", "/app/worker.mjs"] - volumeMounts: - - name: script - mountPath: /app/worker.mjs - subPath: worker.mjs - readOnly: true - volumes: - - name: script - configMap: - name: dlob-worker-script diff --git a/kustomize/base/dlob-worker/worker.mjs b/kustomize/base/dlob-worker/worker.mjs deleted file mode 100644 index 5a9220d..0000000 --- a/kustomize/base/dlob-worker/worker.mjs +++ /dev/null @@ -1,435 +0,0 @@ -import fs from 'node:fs'; -import * as http from 'node:http'; -import * as https from 'node:https'; -import process from 'node:process'; -import { setTimeout as sleep } from 'node:timers/promises'; - -function readJsonFile(filePath) { - try { - const raw = fs.readFileSync(filePath, 'utf8'); - return JSON.parse(raw); - } catch { - return undefined; - } -} - -function getIsoNow() { - return new Date().toISOString(); -} - -function clampInt(value, min, max, fallback) { - const n = Number.parseInt(String(value ?? ''), 10); - if (!Number.isInteger(n)) return fallback; - return Math.min(max, Math.max(min, n)); -} - -function envList(name, fallbackCsv) { - const raw = process.env[name] ?? fallbackCsv; - return String(raw) - .split(',') - .map((s) => s.trim()) - .filter(Boolean); -} - -function envBool(name, fallback = false) { - const raw = process.env[name]; - if (raw == null) return fallback; - const v = String(raw).trim().toLowerCase(); - if (['1', 'true', 'yes', 'y', 'on'].includes(v)) return true; - if (['0', 'false', 'no', 'n', 'off'].includes(v)) return false; - return fallback; -} - -function resolveConfig() { - const tokensPath = - process.env.HASURA_TOKENS_FILE || - process.env.TOKENS_FILE || - process.env.HASURA_CONFIG_FILE || - '/app/tokens/hasura.json'; - const tokens = readJsonFile(tokensPath) || {}; - - const hasuraUrl = - process.env.HASURA_GRAPHQL_URL || - tokens.graphqlUrl || - tokens.apiUrl || - 'http://hasura:8080/v1/graphql'; - const hasuraAdminSecret = - process.env.HASURA_ADMIN_SECRET || - process.env.HASURA_GRAPHQL_ADMIN_SECRET || - tokens.adminSecret || - tokens.hasuraAdminSecret; - const hasuraAuthToken = process.env.HASURA_AUTH_TOKEN || process.env.HASURA_JWT || undefined; - - const dlobHttpBase = String(process.env.DLOB_HTTP_URL || process.env.DLOB_HTTP_BASE || 'https://dlob.drift.trade') - .trim() - .replace(/\/$/, ''); - const dlobForceIpv6 = envBool('DLOB_FORCE_IPV6', false); - const dlobSource = String(process.env.DLOB_SOURCE || 'mevnode').trim() || 'mevnode'; - - const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP'); - const depth = clampInt(process.env.DLOB_DEPTH, 1, 50, 10); - const pollMs = clampInt(process.env.DLOB_POLL_MS, 100, 10_000, 500); - - const pricePrecision = Number(process.env.PRICE_PRECISION || 1_000_000); - const basePrecision = Number(process.env.BASE_PRECISION || 1_000_000_000); - if (!Number.isFinite(pricePrecision) || pricePrecision <= 0) - throw new Error(`Invalid PRICE_PRECISION: ${process.env.PRICE_PRECISION}`); - if (!Number.isFinite(basePrecision) || basePrecision <= 0) - throw new Error(`Invalid BASE_PRECISION: ${process.env.BASE_PRECISION}`); - - return { - hasuraUrl, - hasuraAdminSecret, - hasuraAuthToken, - dlobSource, - dlobHttpBase, - dlobForceIpv6, - markets, - depth, - pollMs, - pricePrecision, - basePrecision, - }; -} - -async function requestText(url, { timeoutMs, family } = {}) { - const u = new URL(url); - const client = u.protocol === 'https:' ? https : http; - - const port = u.port ? Number.parseInt(u.port, 10) : u.protocol === 'https:' ? 443 : 80; - if (!Number.isFinite(port)) throw new Error(`Invalid port for url: ${url}`); - - return await new Promise((resolve, reject) => { - const req = client.request( - { - protocol: u.protocol, - hostname: u.hostname, - port, - path: `${u.pathname}${u.search}`, - method: 'GET', - family, - servername: u.hostname, - headers: { - accept: 'application/json', - }, - }, - (res) => { - let data = ''; - res.setEncoding('utf8'); - res.on('data', (chunk) => { - data += chunk; - }); - res.on('end', () => { - resolve({ status: res.statusCode ?? 0, text: data }); - }); - } - ); - - req.on('error', reject); - req.setTimeout(timeoutMs ?? 5_000, () => { - req.destroy(new Error(`Timeout after ${timeoutMs ?? 5_000}ms`)); - }); - req.end(); - }); -} - -async function graphqlRequest(cfg, query, variables) { - const headers = { 'content-type': 'application/json' }; - if (cfg.hasuraAuthToken) { - headers.authorization = `Bearer ${cfg.hasuraAuthToken}`; - } else if (cfg.hasuraAdminSecret) { - headers['x-hasura-admin-secret'] = cfg.hasuraAdminSecret; - } else { - throw new Error('Missing Hasura auth (set HASURA_AUTH_TOKEN or HASURA_ADMIN_SECRET or mount tokens/hasura.json)'); - } - - const res = await fetch(cfg.hasuraUrl, { - method: 'POST', - headers, - body: JSON.stringify({ query, variables }), - signal: AbortSignal.timeout(10_000), - }); - const text = await res.text(); - if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`); - const json = JSON.parse(text); - if (json.errors?.length) { - throw new Error(json.errors.map((e) => e.message).join(' | ')); - } - return json.data; -} - -function toNumberOrNull(value) { - if (value == null) return null; - if (typeof value === 'number') return Number.isFinite(value) ? value : null; - if (typeof value === 'string') { - const s = value.trim(); - if (!s) return null; - const n = Number(s); - return Number.isFinite(n) ? n : null; - } - return null; -} - -function numStr(value) { - if (value == null) return null; - if (typeof value === 'number') return Number.isFinite(value) ? String(value) : null; - if (typeof value === 'string') return value.trim() || null; - return null; -} - -function parseScaled(valueRaw, scale) { - const n = toNumberOrNull(valueRaw); - if (n == null) return null; - return n / scale; -} - -function computeStats({ l2, depth, pricePrecision, basePrecision }) { - const bids = Array.isArray(l2?.bids) ? l2.bids : []; - const asks = Array.isArray(l2?.asks) ? l2.asks : []; - - const bestBid = parseScaled(l2?.bestBidPrice ?? bids?.[0]?.price, pricePrecision); - const bestAsk = parseScaled(l2?.bestAskPrice ?? asks?.[0]?.price, pricePrecision); - const markPrice = parseScaled(l2?.markPrice, pricePrecision); - const oraclePrice = parseScaled(l2?.oracleData?.price ?? l2?.oracle, pricePrecision); - - const mid = bestBid != null && bestAsk != null ? (bestBid + bestAsk) / 2 : null; - const spreadAbs = bestBid != null && bestAsk != null ? bestAsk - bestBid : null; - const spreadBps = spreadAbs != null && mid != null && mid > 0 ? (spreadAbs / mid) * 10_000 : null; - - const levels = Math.max(1, depth); - let bidBase = 0; - let askBase = 0; - let bidUsd = 0; - let askUsd = 0; - - for (let i = 0; i < Math.min(levels, bids.length); i += 1) { - const p = parseScaled(bids[i]?.price, pricePrecision); - const s = toNumberOrNull(bids[i]?.size); - if (p == null || s == null) continue; - const base = s / basePrecision; - bidBase += base; - bidUsd += base * p; - } - - for (let i = 0; i < Math.min(levels, asks.length); i += 1) { - const p = parseScaled(asks[i]?.price, pricePrecision); - const s = toNumberOrNull(asks[i]?.size); - if (p == null || s == null) continue; - const base = s / basePrecision; - askBase += base; - askUsd += base * p; - } - - const denom = bidUsd + askUsd; - const imbalance = denom > 0 ? (bidUsd - askUsd) / denom : null; - - return { - bestBid, - bestAsk, - mid, - spreadAbs, - spreadBps, - markPrice, - oraclePrice, - depthLevels: levels, - bidBase, - askBase, - bidUsd, - askUsd, - imbalance, - }; -} - -function l2ToInsertObject({ dlobSource, l2, updatedAt, pricePrecision }) { - return { - source: dlobSource, - market_name: String(l2.marketName), - market_type: String(l2.marketType || 'perp'), - market_index: typeof l2.marketIndex === 'number' ? l2.marketIndex : null, - ts: l2.ts == null ? null : String(l2.ts), - slot: l2.slot == null ? null : String(l2.slot), - mark_price: numStr(parseScaled(l2.markPrice, pricePrecision)), - oracle_price: numStr(parseScaled(l2.oracleData?.price ?? l2.oracle, pricePrecision)), - best_bid_price: numStr(parseScaled(l2.bestBidPrice, pricePrecision)), - best_ask_price: numStr(parseScaled(l2.bestAskPrice, pricePrecision)), - bids: l2.bids ?? null, - asks: l2.asks ?? null, - raw: l2 ?? null, - updated_at: updatedAt, - }; -} - -function statsToInsertObject({ dlobSource, l2, stats, updatedAt }) { - return { - source: dlobSource, - market_name: String(l2.marketName), - market_type: String(l2.marketType || 'perp'), - market_index: typeof l2.marketIndex === 'number' ? l2.marketIndex : null, - ts: l2.ts == null ? null : String(l2.ts), - slot: l2.slot == null ? null : String(l2.slot), - mark_price: stats.markPrice == null ? null : String(stats.markPrice), - oracle_price: stats.oraclePrice == null ? null : String(stats.oraclePrice), - best_bid_price: stats.bestBid == null ? null : String(stats.bestBid), - best_ask_price: stats.bestAsk == null ? null : String(stats.bestAsk), - mid_price: stats.mid == null ? null : String(stats.mid), - spread_abs: stats.spreadAbs == null ? null : String(stats.spreadAbs), - spread_bps: stats.spreadBps == null ? null : String(stats.spreadBps), - depth_levels: stats.depthLevels, - depth_bid_base: Number.isFinite(stats.bidBase) ? String(stats.bidBase) : null, - depth_ask_base: Number.isFinite(stats.askBase) ? String(stats.askBase) : null, - depth_bid_usd: Number.isFinite(stats.bidUsd) ? String(stats.bidUsd) : null, - depth_ask_usd: Number.isFinite(stats.askUsd) ? String(stats.askUsd) : null, - imbalance: stats.imbalance == null ? null : String(stats.imbalance), - raw: { - spreadPct: l2.spreadPct ?? null, - spreadQuote: l2.spreadQuote ?? null, - }, - updated_at: updatedAt, - }; -} - -async function fetchL2(cfg, marketName) { - const u = new URL(`${cfg.dlobHttpBase}/l2`); - u.searchParams.set('marketName', marketName); - u.searchParams.set('depth', String(cfg.depth)); - - const url = u.toString(); - if (cfg.dlobForceIpv6) { - const { status, text } = await requestText(url, { timeoutMs: 5_000, family: 6 }); - if (status < 200 || status >= 300) throw new Error(`DLOB HTTP ${status}: ${text}`); - return JSON.parse(text); - } - - const res = await fetch(url, { signal: AbortSignal.timeout(5_000) }); - const text = await res.text(); - if (!res.ok) throw new Error(`DLOB HTTP ${res.status}: ${text}`); - return JSON.parse(text); -} - -async function upsertBatch(cfg, l2Objects, statsObjects) { - if (!l2Objects.length && !statsObjects.length) return; - - const mutation = ` - mutation UpsertDlob($l2: [dlob_l2_latest_insert_input!]!, $stats: [dlob_stats_latest_insert_input!]!) { - insert_dlob_l2_latest( - objects: $l2 - on_conflict: { - constraint: dlob_l2_latest_pkey - update_columns: [ - market_type - market_index - ts - slot - mark_price - oracle_price - best_bid_price - best_ask_price - bids - asks - raw - updated_at - ] - } - ) { affected_rows } - insert_dlob_stats_latest( - objects: $stats - on_conflict: { - constraint: dlob_stats_latest_pkey - update_columns: [ - market_type - market_index - ts - slot - mark_price - oracle_price - best_bid_price - best_ask_price - mid_price - spread_abs - spread_bps - depth_levels - depth_bid_base - depth_ask_base - depth_bid_usd - depth_ask_usd - imbalance - raw - updated_at - ] - } - ) { affected_rows } - } - `; - - await graphqlRequest(cfg, mutation, { l2: l2Objects, stats: statsObjects }); -} - -async function main() { - const cfg = resolveConfig(); - const lastTsByMarket = new Map(); - - console.log( - JSON.stringify( - { - service: 'dlob-worker', - startedAt: getIsoNow(), - hasuraUrl: cfg.hasuraUrl, - hasuraAuth: cfg.hasuraAuthToken ? 'bearer' : cfg.hasuraAdminSecret ? 'admin-secret' : 'none', - dlobSource: cfg.dlobSource, - dlobHttpBase: cfg.dlobHttpBase, - dlobForceIpv6: cfg.dlobForceIpv6, - markets: cfg.markets, - depth: cfg.depth, - pollMs: cfg.pollMs, - }, - null, - 2 - ) - ); - - while (true) { - const updatedAt = getIsoNow(); - - const results = await Promise.allSettled(cfg.markets.map((m) => fetchL2(cfg, m))); - const l2Objects = []; - const statsObjects = []; - - for (let i = 0; i < results.length; i += 1) { - const market = cfg.markets[i]; - const r = results[i]; - if (r.status !== 'fulfilled') { - console.error(`[dlob-worker] fetch ${market}: ${String(r.reason?.message || r.reason)}`); - continue; - } - const l2 = r.value; - if (!l2?.marketName) continue; - - const ts = l2.ts == null ? null : String(l2.ts); - if (ts != null && lastTsByMarket.get(l2.marketName) === ts) continue; - if (ts != null) lastTsByMarket.set(l2.marketName, ts); - - const stats = computeStats({ - l2, - depth: cfg.depth, - pricePrecision: cfg.pricePrecision, - basePrecision: cfg.basePrecision, - }); - - l2Objects.push(l2ToInsertObject({ dlobSource: cfg.dlobSource, l2, updatedAt, pricePrecision: cfg.pricePrecision })); - statsObjects.push(statsToInsertObject({ dlobSource: cfg.dlobSource, l2, stats, updatedAt })); - } - - try { - await upsertBatch(cfg, l2Objects, statsObjects); - } catch (err) { - console.error(`[dlob-worker] upsert: ${String(err?.message || err)}`); - } - - await sleep(cfg.pollMs); - } -} - -main().catch((err) => { - console.error(String(err?.stack || err)); - process.exitCode = 1; -}); diff --git a/kustomize/base/dlob/publisher-deployment.yaml b/kustomize/base/dlob/publisher-deployment.yaml index d31f762..4aee67a 100644 --- a/kustomize/base/dlob/publisher-deployment.yaml +++ b/kustomize/base/dlob/publisher-deployment.yaml @@ -1,24 +1,24 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: dlob-publisher + name: dlob-publisher-hot annotations: argocd.argoproj.io/sync-wave: "4" spec: replicas: 1 selector: matchLabels: - app.kubernetes.io/name: dlob-publisher + app.kubernetes.io/name: dlob-publisher-hot template: metadata: labels: - app.kubernetes.io/name: dlob-publisher + app.kubernetes.io/name: dlob-publisher-hot spec: imagePullSecrets: - name: gitea-registry containers: - name: publisher - image: gitea.mpabi.pl/trade/trade-dlob-server:sha-5d55631-canonical + image: gitea.mpabi.pl/trade/trade-dlob-server:grpc-reconnect-20260319-023351 imagePullPolicy: IfNotPresent ports: - name: http @@ -41,7 +41,7 @@ spec: - name: ELASTICACHE_PORT value: "6379" - name: REDIS_CLIENT - value: DLOB + value: DLOB_HOT - name: PERP_MARKETS_TO_LOAD value: "0,7,24" - name: ENDPOINT diff --git a/kustomize/base/dlob/server-deployment.yaml b/kustomize/base/dlob/server-deployment.yaml deleted file mode 100644 index 6b65fa7..0000000 --- a/kustomize/base/dlob/server-deployment.yaml +++ /dev/null @@ -1,63 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dlob-server - annotations: - argocd.argoproj.io/sync-wave: "4" -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: dlob-server - template: - metadata: - labels: - app.kubernetes.io/name: dlob-server - spec: - imagePullSecrets: - - name: gitea-registry - containers: - - name: server - image: gitea.mpabi.pl/trade/trade-dlob-server:sha-5d55631-canonical - imagePullPolicy: IfNotPresent - ports: - - name: http - containerPort: 6969 - env: - - name: RUNNING_LOCAL - value: "true" - - name: LOCAL_CACHE - value: "true" - - name: ENV - value: mainnet-beta - - name: PORT - value: "6969" - - name: ELASTICACHE_HOST - value: dlob-redis - - name: ELASTICACHE_PORT - value: "6379" - - name: REDIS_CLIENT - value: DLOB - - name: ENDPOINT - valueFrom: - secretKeyRef: - name: trade-dlob-rpc - key: ENDPOINT - - name: WS_ENDPOINT - valueFrom: - secretKeyRef: - name: trade-dlob-rpc - key: WS_ENDPOINT - command: ["node", "/lib/serverLite.js"] - readinessProbe: - httpGet: - path: /startup - port: http - initialDelaySeconds: 5 - periodSeconds: 10 - livenessProbe: - httpGet: - path: /health - port: http - initialDelaySeconds: 20 - periodSeconds: 20 diff --git a/kustomize/base/dlob/server-service.yaml b/kustomize/base/dlob/server-service.yaml deleted file mode 100644 index f423fd5..0000000 --- a/kustomize/base/dlob/server-service.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: dlob-server - annotations: - argocd.argoproj.io/sync-wave: "4" -spec: - selector: - app.kubernetes.io/name: dlob-server - ports: - - name: http - port: 6969 - targetPort: http diff --git a/kustomize/base/frontend/deployment.yaml b/kustomize/base/frontend/deployment.yaml index 32e0a8b..7727eda 100644 --- a/kustomize/base/frontend/deployment.yaml +++ b/kustomize/base/frontend/deployment.yaml @@ -16,7 +16,7 @@ spec: - name: gitea-registry containers: - name: frontend - image: gitea.mpabi.pl/trade/trade-frontend:sha-b06fe7f + image: gitea.mpabi.pl/trade/trade-frontend:pg-derived-phase1-20260319-030252 imagePullPolicy: IfNotPresent ports: - name: http @@ -26,12 +26,8 @@ spec: value: "8081" - name: APP_VERSION value: "staging" - - name: API_UPSTREAM - value: "http://trade-api:8787" - name: BASIC_AUTH_FILE value: "/tokens/frontend.json" - - name: API_READ_TOKEN_FILE - value: "/tokens/read.json" volumeMounts: - name: tokens mountPath: /tokens diff --git a/kustomize/base/hasura/hasura-bootstrap.mjs b/kustomize/base/hasura/hasura-bootstrap.mjs index 2c19cdb..335203d 100644 --- a/kustomize/base/hasura/hasura-bootstrap.mjs +++ b/kustomize/base/hasura/hasura-bootstrap.mjs @@ -92,25 +92,33 @@ async function main() { const PUBLIC_DLOB_SOURCE_HEADER = 'X-Hasura-Dlob-Source'; const apiTokensTable = { schema: 'public', name: 'api_tokens' }; + const botConfigTable = { schema: 'public', name: 'bot_config' }; + const botStateTable = { schema: 'public', name: 'bot_state' }; + const botEventsTable = { schema: 'public', name: 'bot_events' }; const source = 'default'; const baseTicks = { schema: 'public', name: 'drift_ticks' }; - const dlobL2LatestTable = { schema: 'public', name: 'dlob_l2_latest' }; - const dlobL2LatestProjectionView = { schema: 'public', name: 'dlob_l2_latest_projection' }; - const dlobL3LatestProjectionView = { schema: 'public', name: 'dlob_l3_latest_projection' }; - const dlobBestMakersLatestProjectionView = { schema: 'public', name: 'dlob_best_makers_latest_projection' }; - const dlobStatsLatestTable = { schema: 'public', name: 'dlob_stats_latest' }; - const dlobStatsLatestProjectionView = { schema: 'public', name: 'dlob_stats_latest_projection' }; - const dlobDepthBpsLatestTable = { schema: 'public', name: 'dlob_depth_bps_latest' }; - const dlobSlippageLatestTable = { schema: 'public', name: 'dlob_slippage_latest' }; - const dlobSlippageLatestV2Table = { schema: 'public', name: 'dlob_slippage_latest_v2' }; + const dlobHotSnapshotLatestTable = { schema: 'public', name: 'dlob_hot_snapshot_latest' }; + const dlobHotDerivedLatestTable = { schema: 'public', name: 'dlob_hot_derived_latest' }; + const dlobAllDerivedLatestTable = { schema: 'public', name: 'dlob_all_derived_latest' }; const candlesCacheTable = { schema: 'public', name: 'drift_candles_cache' }; - const dlobStatsTsTable = { schema: 'public', name: 'dlob_stats_ts' }; - const dlobDepthBpsTsTable = { schema: 'public', name: 'dlob_depth_bps_ts' }; - const dlobSlippageTsTable = { schema: 'public', name: 'dlob_slippage_ts' }; - const dlobSlippageTsV2Table = { schema: 'public', name: 'dlob_slippage_ts_v2' }; + const dlobHotDerivedTsTable = { schema: 'public', name: 'dlob_hot_derived_ts' }; + const dlobAllDerivedTsTable = { schema: 'public', name: 'dlob_all_derived_ts' }; const baseCandlesFn = { schema: 'public', name: 'get_drift_candles' }; const candlesReturnTable = { schema: 'public', name: 'drift_candles' }; + const legacyDlobRelations = [ + { schema: 'public', name: 'dlob_l2_latest' }, + { schema: 'public', name: 'dlob_l2_latest_projection' }, + { schema: 'public', name: 'dlob_stats_latest' }, + { schema: 'public', name: 'dlob_stats_latest_projection' }, + { schema: 'public', name: 'dlob_depth_bps_latest' }, + { schema: 'public', name: 'dlob_slippage_latest' }, + { schema: 'public', name: 'dlob_slippage_latest_v2' }, + { schema: 'public', name: 'dlob_stats_ts' }, + { schema: 'public', name: 'dlob_depth_bps_ts' }, + { schema: 'public', name: 'dlob_slippage_ts' }, + { schema: 'public', name: 'dlob_slippage_ts_v2' }, + ]; const extraTicksName = normalizeName(TARGET_TICKS_TABLE); const extraCandlesName = normalizeName(TARGET_CANDLES_FUNCTION); @@ -293,139 +301,174 @@ async function main() { const dlobPublicFilter = { source: { _eq: PUBLIC_DLOB_SOURCE_HEADER } }; - await metadataIgnore({ type: 'pg_untrack_table', args: { source, table: dlobL2LatestTable, cascade: true } }); - await ensureProjectedPublicSelectTable(dlobL2LatestProjectionView, [ + for (const table of legacyDlobRelations) { + await metadataIgnore({ type: 'pg_untrack_table', args: { source, table, cascade: true } }); + } + + await ensurePublicSelectTable(dlobHotSnapshotLatestTable, [ 'source', - 'market_name', + 'redis_key', + 'snapshot_kind', 'market_type', 'market_index', - 'ts', + 'market_name', + 'is_indicative', + 'ts_ms', 'slot', - 'mark_price', - 'oracle_price', - 'best_bid_price', - 'best_ask_price', + 'market_slot', + 'payload_hash', + 'mark_price_raw', + 'oracle_price_raw', + 'best_bid_price_raw', + 'best_ask_price_raw', + 'spread_pct_raw', + 'spread_quote_raw', + 'oracle_data', 'bids', 'asks', - 'raw', + 'payload', 'updated_at', - ], { publicFilter: dlobPublicFilter, customName: 'dlob_l2_latest' }); + ]); - await metadataIgnore({ type: 'pg_untrack_table', args: { source, table: dlobStatsLatestTable, cascade: true } }); - await ensureProjectedPublicSelectTable(dlobStatsLatestProjectionView, [ + await ensurePublicSelectTable(dlobHotDerivedLatestTable, [ 'source', - 'market_name', 'market_type', 'market_index', - 'ts', + 'market_name', + 'is_indicative', + 'ts_ms', 'slot', + 'market_slot', 'mark_price', 'oracle_price', 'best_bid_price', 'best_ask_price', 'mid_price', - 'spread_abs', + 'spread_quote', 'spread_bps', 'depth_levels', + 'bid_levels', + 'ask_levels', + 'top_bid_size', + 'top_ask_size', + 'top_bid_notional', + 'top_ask_notional', 'depth_bid_base', 'depth_ask_base', - 'depth_bid_usd', - 'depth_ask_usd', + 'depth_bid_quote', + 'depth_ask_quote', 'imbalance', - 'raw', + 'bids_norm', + 'asks_norm', + 'raw_payload_hash', 'updated_at', - ], { publicFilter: dlobPublicFilter, customName: 'dlob_stats_latest' }); + ]); - await ensureProjectedPublicSelectTable(dlobL3LatestProjectionView, [ + await ensurePublicSelectTable(dlobAllDerivedLatestTable, [ 'source', - 'market_name', 'market_type', 'market_index', - 'ts', - 'slot', - 'bids', - 'asks', - 'raw', - 'updated_at', - ], { publicFilter: dlobPublicFilter }); - - await ensureProjectedPublicSelectTable(dlobBestMakersLatestProjectionView, [ - 'source', 'market_name', - 'market_type', - 'market_index', + 'is_indicative', + 'ts_ms', 'slot', - 'bids', - 'asks', - 'raw', - 'updated_at', - ], { publicFilter: dlobPublicFilter }); - - await ensurePublicSelectTable(dlobDepthBpsLatestTable, [ - 'source', - 'market_name', - 'band_bps', - 'market_type', - 'market_index', - 'ts', - 'slot', - 'mid_price', + 'market_slot', + 'mark_price', + 'oracle_price', 'best_bid_price', 'best_ask_price', - 'bid_base', - 'ask_base', - 'bid_usd', - 'ask_usd', + 'mid_price', + 'spread_quote', + 'spread_bps', + 'depth_levels', + 'bid_levels', + 'ask_levels', + 'top_bid_size', + 'top_ask_size', + 'top_bid_notional', + 'top_ask_notional', + 'depth_bid_base', + 'depth_ask_base', + 'depth_bid_quote', + 'depth_ask_quote', 'imbalance', - 'raw', + 'bids_norm', + 'asks_norm', + 'raw_payload_hash', 'updated_at', - ], { publicFilter: dlobPublicFilter }); + ]); - await ensurePublicSelectTable(dlobSlippageLatestTable, [ + await ensurePublicSelectTable(dlobHotDerivedTsTable, [ + 'event_ts', + 'id', 'source', - 'market_name', - 'side', - 'size_usd', 'market_type', 'market_index', - 'ts', + 'market_name', + 'is_indicative', + 'ts_ms', 'slot', - 'mid_price', + 'market_slot', + 'mark_price', + 'oracle_price', 'best_bid_price', 'best_ask_price', - 'vwap_price', - 'worst_price', - 'filled_usd', - 'filled_base', - 'impact_bps', - 'levels_consumed', - 'fill_pct', - 'raw', - 'updated_at', - ], { publicFilter: dlobPublicFilter }); + 'mid_price', + 'spread_quote', + 'spread_bps', + 'depth_levels', + 'bid_levels', + 'ask_levels', + 'top_bid_size', + 'top_ask_size', + 'top_bid_notional', + 'top_ask_notional', + 'depth_bid_base', + 'depth_ask_base', + 'depth_bid_quote', + 'depth_ask_quote', + 'imbalance', + 'bids_norm', + 'asks_norm', + 'raw_payload_hash', + 'inserted_at', + ]); - await ensurePublicSelectTable(dlobSlippageLatestV2Table, [ + await ensurePublicSelectTable(dlobAllDerivedTsTable, [ + 'event_ts', + 'id', 'source', - 'market_name', - 'side', - 'size_usd', 'market_type', 'market_index', - 'ts', + 'market_name', + 'is_indicative', + 'ts_ms', 'slot', - 'mid_price', + 'market_slot', + 'mark_price', + 'oracle_price', 'best_bid_price', 'best_ask_price', - 'vwap_price', - 'worst_price', - 'filled_usd', - 'filled_base', - 'impact_bps', - 'levels_consumed', - 'fill_pct', - 'raw', - 'updated_at', - ], { publicFilter: dlobPublicFilter }); + 'mid_price', + 'spread_quote', + 'spread_bps', + 'depth_levels', + 'bid_levels', + 'ask_levels', + 'top_bid_size', + 'top_ask_size', + 'top_bid_notional', + 'top_ask_notional', + 'depth_bid_base', + 'depth_ask_base', + 'depth_bid_quote', + 'depth_ask_quote', + 'imbalance', + 'bids_norm', + 'asks_norm', + 'raw_payload_hash', + 'inserted_at', + ]); await ensurePublicSelectTable(dlobStatsTsTable, [ 'ts', @@ -517,6 +560,12 @@ async function main() { 'raw', ], { publicFilter: dlobPublicFilter }); + // Bot control-plane tables (tracked; permissions are intentionally not created here by default). + for (const t of [botConfigTable, botStateTable, botEventsTable]) { + await metadataIgnore({ type: 'pg_untrack_table', args: { source, table: t } }); + await metadata({ type: 'pg_track_table', args: { source, table: t } }); + } + // Return table type for candle functions (needed for Hasura to track the function). await metadataIgnore({ type: 'pg_track_table', args: { source, table: candlesReturnTable } }); @@ -532,7 +581,7 @@ async function main() { } for (const fn of candleFns) { - // Function for aggregated candle queries (used by trade-api). + // Function for aggregated candle queries and indicator reads over Hasura. await metadataIgnore({ type: 'pg_untrack_function', args: { source, function: fn } }); await metadata({ type: 'pg_track_function', args: { source, function: fn } }); } diff --git a/kustomize/base/ingestor/deployment.yaml b/kustomize/base/ingestor/deployment.yaml deleted file mode 100644 index d1498d5..0000000 --- a/kustomize/base/ingestor/deployment.yaml +++ /dev/null @@ -1,49 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trade-ingestor -spec: - replicas: 1 - selector: - matchLabels: - app.kubernetes.io/name: trade-ingestor - template: - metadata: - labels: - app.kubernetes.io/name: trade-ingestor - spec: - imagePullSecrets: - - name: gitea-registry - containers: - - name: ingestor - image: gitea.mpabi.pl/trade/trade-ingestor:k3s-20260106013603 - imagePullPolicy: IfNotPresent - env: - - name: MARKET_NAME - value: "PUMP-PERP" - - name: INTERVAL_MS - value: "1000" - - name: SOURCE - value: "drift_oracle" - - name: INGEST_VIA - value: "api" - - name: INGEST_API_URL - value: "http://trade-api:8787" - volumeMounts: - - name: tokens - mountPath: /app/tokens - readOnly: true - command: - - sh - - -c - args: - - > - npm run ingest:oracle -- \ - --market-name "${MARKET_NAME}" \ - --interval-ms "${INTERVAL_MS}" \ - --source "${SOURCE}" \ - --ingest-via "${INGEST_VIA}" - volumes: - - name: tokens - secret: - secretName: trade-ingestor-tokens diff --git a/kustomize/base/initdb/001_init.sql b/kustomize/base/initdb/001_init.sql index 70f9f19..197afc3 100644 --- a/kustomize/base/initdb/001_init.sql +++ b/kustomize/base/initdb/001_init.sql @@ -111,8 +111,52 @@ ALTER TABLE api_tokens ADD COLUMN IF NOT EXISTS scopes TEXT[] NOT NULL DEFAULT A CREATE INDEX IF NOT EXISTS api_tokens_revoked_at_idx ON api_tokens (revoked_at); +-- Bot control-plane (desired state) + executor telemetry (state/events). +-- +-- MVP intent: +-- - `bot_config`: desired state (mode/market/limits/kill switch) +-- - `bot_state`: heartbeat + last error/action +-- - `bot_events`: append-only audit log +CREATE TABLE IF NOT EXISTS public.bot_config ( + id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL UNIQUE, + market_name TEXT NOT NULL, + market_type TEXT NOT NULL DEFAULT 'perp', + mode TEXT NOT NULL DEFAULT 'off', + kill_switch BOOLEAN NOT NULL DEFAULT FALSE, + params JSONB, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS bot_config_updated_at_desc_idx + ON public.bot_config (updated_at DESC); + +CREATE TABLE IF NOT EXISTS public.bot_state ( + bot_id UUID PRIMARY KEY REFERENCES public.bot_config(id) ON DELETE CASCADE, + last_heartbeat_at TIMESTAMPTZ, + last_action_at TIMESTAMPTZ, + last_error TEXT, + state JSONB, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE INDEX IF NOT EXISTS bot_state_updated_at_desc_idx + ON public.bot_state (updated_at DESC); + +CREATE TABLE IF NOT EXISTS public.bot_events ( + id BIGSERIAL PRIMARY KEY, + bot_id UUID NOT NULL REFERENCES public.bot_config(id) ON DELETE CASCADE, + ts TIMESTAMPTZ NOT NULL DEFAULT now(), + type TEXT NOT NULL, + payload JSONB +); + +CREATE INDEX IF NOT EXISTS bot_events_bot_ts_desc_idx + ON public.bot_events (bot_id, ts DESC); + -- Compute OHLC candles from `drift_ticks` for a symbol and bucket size. --- Exposed via Hasura (track function) and used by trade-api to compute indicators server-side. +-- Exposed via Hasura (track function) for server-side candle aggregation and indicator reads. -- Hasura tracks functions only if they return SETOF a table/view type. -- This table is used purely as the return type for candle functions. CREATE TABLE IF NOT EXISTS public.drift_candles ( @@ -838,41 +882,8 @@ CREATE TABLE IF NOT EXISTS public.dlob_orderbook_l2_latest ( PRIMARY KEY (source, market_name, is_indicative) ); -CREATE TABLE IF NOT EXISTS public.dlob_orderbook_l3_latest ( - source TEXT NOT NULL, - market_name TEXT NOT NULL, - market_type TEXT NOT NULL, - market_index INTEGER, - is_indicative BOOLEAN NOT NULL DEFAULT FALSE, - ts BIGINT, - slot BIGINT, - oracle_data JSONB, - bids JSONB, - asks JSONB, - payload JSONB NOT NULL, - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - PRIMARY KEY (source, market_name, is_indicative) -); - -CREATE TABLE IF NOT EXISTS public.dlob_best_makers_latest ( - source TEXT NOT NULL, - market_name TEXT NOT NULL, - market_type TEXT NOT NULL, - market_index INTEGER, - slot BIGINT, - bids JSONB, - asks JSONB, - payload JSONB NOT NULL, - updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), - PRIMARY KEY (source, market_name) -); - CREATE INDEX IF NOT EXISTS dlob_orderbook_l2_latest_updated_at_idx ON public.dlob_orderbook_l2_latest (updated_at DESC); -CREATE INDEX IF NOT EXISTS dlob_orderbook_l3_latest_updated_at_idx - ON public.dlob_orderbook_l3_latest (updated_at DESC); -CREATE INDEX IF NOT EXISTS dlob_best_makers_latest_updated_at_idx - ON public.dlob_best_makers_latest (updated_at DESC); CREATE OR REPLACE VIEW public.dlob_l2_latest_projection AS SELECT @@ -905,34 +916,6 @@ SELECT FROM public.dlob_orderbook_l2_latest WHERE is_indicative = FALSE; -CREATE OR REPLACE VIEW public.dlob_l3_latest_projection AS -SELECT - source, - market_name, - market_type, - market_index, - ts, - slot, - bids, - asks, - payload AS raw, - updated_at -FROM public.dlob_orderbook_l3_latest -WHERE is_indicative = FALSE; - -CREATE OR REPLACE VIEW public.dlob_best_makers_latest_projection AS -SELECT - source, - market_name, - market_type, - market_index, - slot, - bids, - asks, - payload AS raw, - updated_at -FROM public.dlob_best_makers_latest; - CREATE OR REPLACE VIEW public.dlob_stats_latest_projection AS WITH base AS ( SELECT @@ -1057,3 +1040,286 @@ SELECT FROM base b LEFT JOIN bid_levels bids USING (source, market_name) LEFT JOIN ask_levels asks USING (source, market_name); + +-- Retired legacy DLOB read-side pipeline artifacts. Keep this cleanup in initdb so +-- fresh volumes and reused volumes both converge on the current Redis -> derived tables workflow. +DROP VIEW IF EXISTS public.dlob_l2_latest_projection; +DROP VIEW IF EXISTS public.dlob_stats_latest_projection; +DROP TABLE IF EXISTS public.dlob_l2_latest; +DROP TABLE IF EXISTS public.dlob_stats_latest; +DROP TABLE IF EXISTS public.dlob_depth_bps_latest; +DROP TABLE IF EXISTS public.dlob_slippage_latest; +DROP TABLE IF EXISTS public.dlob_slippage_latest_v2; +DROP TABLE IF EXISTS public.dlob_stats_ts; +DROP TABLE IF EXISTS public.dlob_depth_bps_ts; +DROP TABLE IF EXISTS public.dlob_slippage_ts; +DROP TABLE IF EXISTS public.dlob_slippage_ts_v2; +DROP TABLE IF EXISTS public.dlob_orderbook_l2_latest; + +-- Canonical raw DLOB hot snapshots written from Redis to PostgreSQL. +CREATE TABLE IF NOT EXISTS public.dlob_hot_snapshot_latest ( + source TEXT NOT NULL, + redis_key TEXT NOT NULL, + snapshot_kind TEXT NOT NULL CHECK (snapshot_kind IN ('orderbook_l2', 'orderbook_l3', 'best_makers')), + market_type TEXT NOT NULL, + market_index INTEGER NOT NULL, + market_name TEXT NOT NULL, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts_ms BIGINT NOT NULL, + slot BIGINT, + market_slot BIGINT, + payload_hash TEXT NOT NULL, + mark_price_raw TEXT, + oracle_price_raw TEXT, + best_bid_price_raw TEXT, + best_ask_price_raw TEXT, + spread_pct_raw TEXT, + spread_quote_raw TEXT, + oracle_data JSONB, + bids JSONB, + asks JSONB, + payload JSONB NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (source, market_type, market_index, snapshot_kind, is_indicative) +); + +CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_latest_updated_at_idx + ON public.dlob_hot_snapshot_latest (updated_at DESC); + +CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_latest_source_market_idx + ON public.dlob_hot_snapshot_latest (source, market_type, market_index, updated_at DESC); + +CREATE TABLE IF NOT EXISTS public.dlob_hot_snapshot_ts ( + event_ts TIMESTAMPTZ NOT NULL, + id BIGSERIAL NOT NULL, + source TEXT NOT NULL, + redis_key TEXT NOT NULL, + snapshot_kind TEXT NOT NULL CHECK (snapshot_kind IN ('orderbook_l2', 'orderbook_l3', 'best_makers')), + market_type TEXT NOT NULL, + market_index INTEGER NOT NULL, + market_name TEXT NOT NULL, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts_ms BIGINT NOT NULL, + slot BIGINT, + market_slot BIGINT, + payload_hash TEXT NOT NULL, + mark_price_raw TEXT, + oracle_price_raw TEXT, + best_bid_price_raw TEXT, + best_ask_price_raw TEXT, + spread_pct_raw TEXT, + spread_quote_raw TEXT, + oracle_data JSONB, + bids JSONB, + asks JSONB, + payload JSONB NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (event_ts, id) +); + +SELECT create_hypertable('dlob_hot_snapshot_ts', 'event_ts', if_not_exists => TRUE, migrate_data => TRUE); + +CREATE UNIQUE INDEX IF NOT EXISTS dlob_hot_snapshot_ts_dedupe_idx + ON public.dlob_hot_snapshot_ts (event_ts, source, redis_key, payload_hash); + +CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_ts_market_ts_desc_idx + ON public.dlob_hot_snapshot_ts (market_type, market_index, event_ts DESC); + +CREATE INDEX IF NOT EXISTS dlob_hot_snapshot_ts_source_market_ts_desc_idx + ON public.dlob_hot_snapshot_ts (source, market_type, market_index, event_ts DESC); + +CREATE TABLE IF NOT EXISTS public.dlob_hot_derived_latest ( + source TEXT NOT NULL, + market_type TEXT NOT NULL, + market_index INTEGER NOT NULL, + market_name TEXT NOT NULL, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts_ms BIGINT NOT NULL, + slot BIGINT, + market_slot BIGINT, + mark_price NUMERIC, + oracle_price NUMERIC, + best_bid_price NUMERIC, + best_ask_price NUMERIC, + mid_price NUMERIC, + spread_quote NUMERIC, + spread_bps NUMERIC, + depth_levels INTEGER NOT NULL, + bid_levels INTEGER NOT NULL, + ask_levels INTEGER NOT NULL, + top_bid_size NUMERIC, + top_ask_size NUMERIC, + top_bid_notional NUMERIC, + top_ask_notional NUMERIC, + depth_bid_base NUMERIC NOT NULL DEFAULT 0, + depth_ask_base NUMERIC NOT NULL DEFAULT 0, + depth_bid_quote NUMERIC NOT NULL DEFAULT 0, + depth_ask_quote NUMERIC NOT NULL DEFAULT 0, + imbalance NUMERIC, + bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + raw_payload_hash TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (source, market_type, market_index, is_indicative) +); + +CREATE INDEX IF NOT EXISTS dlob_hot_derived_latest_updated_at_idx + ON public.dlob_hot_derived_latest (updated_at DESC); + +CREATE INDEX IF NOT EXISTS dlob_hot_derived_latest_source_market_idx + ON public.dlob_hot_derived_latest (source, market_type, market_index, updated_at DESC); + +CREATE TABLE IF NOT EXISTS public.dlob_hot_derived_ts ( + event_ts TIMESTAMPTZ NOT NULL, + id BIGSERIAL NOT NULL, + source TEXT NOT NULL, + market_type TEXT NOT NULL, + market_index INTEGER NOT NULL, + market_name TEXT NOT NULL, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts_ms BIGINT NOT NULL, + slot BIGINT, + market_slot BIGINT, + mark_price NUMERIC, + oracle_price NUMERIC, + best_bid_price NUMERIC, + best_ask_price NUMERIC, + mid_price NUMERIC, + spread_quote NUMERIC, + spread_bps NUMERIC, + depth_levels INTEGER NOT NULL, + bid_levels INTEGER NOT NULL, + ask_levels INTEGER NOT NULL, + top_bid_size NUMERIC, + top_ask_size NUMERIC, + top_bid_notional NUMERIC, + top_ask_notional NUMERIC, + depth_bid_base NUMERIC NOT NULL DEFAULT 0, + depth_ask_base NUMERIC NOT NULL DEFAULT 0, + depth_bid_quote NUMERIC NOT NULL DEFAULT 0, + depth_ask_quote NUMERIC NOT NULL DEFAULT 0, + imbalance NUMERIC, + bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + raw_payload_hash TEXT NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (event_ts, id) +); + +SELECT create_hypertable('dlob_hot_derived_ts', 'event_ts', if_not_exists => TRUE, migrate_data => TRUE); + +CREATE UNIQUE INDEX IF NOT EXISTS dlob_hot_derived_ts_dedupe_idx + ON public.dlob_hot_derived_ts (event_ts, source, market_type, market_index, is_indicative, raw_payload_hash); + +CREATE INDEX IF NOT EXISTS dlob_hot_derived_ts_market_ts_desc_idx + ON public.dlob_hot_derived_ts (market_type, market_index, event_ts DESC); + +CREATE INDEX IF NOT EXISTS dlob_hot_derived_ts_source_market_ts_desc_idx + ON public.dlob_hot_derived_ts (source, market_type, market_index, event_ts DESC); + +DO $$ +BEGIN + PERFORM add_retention_policy('dlob_hot_snapshot_ts', INTERVAL '30 days'); +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; + +DO $$ +BEGIN + PERFORM add_retention_policy('dlob_hot_derived_ts', INTERVAL '180 days'); +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; + +CREATE TABLE IF NOT EXISTS public.dlob_all_derived_latest ( + source TEXT NOT NULL, + market_type TEXT NOT NULL, + market_index INTEGER NOT NULL, + market_name TEXT NOT NULL, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts_ms BIGINT NOT NULL, + slot BIGINT, + market_slot BIGINT, + mark_price NUMERIC, + oracle_price NUMERIC, + best_bid_price NUMERIC, + best_ask_price NUMERIC, + mid_price NUMERIC, + spread_quote NUMERIC, + spread_bps NUMERIC, + depth_levels INTEGER NOT NULL, + bid_levels INTEGER NOT NULL, + ask_levels INTEGER NOT NULL, + top_bid_size NUMERIC, + top_ask_size NUMERIC, + top_bid_notional NUMERIC, + top_ask_notional NUMERIC, + depth_bid_base NUMERIC NOT NULL DEFAULT 0, + depth_ask_base NUMERIC NOT NULL DEFAULT 0, + depth_bid_quote NUMERIC NOT NULL DEFAULT 0, + depth_ask_quote NUMERIC NOT NULL DEFAULT 0, + imbalance NUMERIC, + bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + raw_payload_hash TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (source, market_type, market_index, is_indicative) +); + +CREATE INDEX IF NOT EXISTS dlob_all_derived_latest_updated_at_idx + ON public.dlob_all_derived_latest (updated_at DESC); + +CREATE INDEX IF NOT EXISTS dlob_all_derived_latest_source_market_idx + ON public.dlob_all_derived_latest (source, market_type, market_index, updated_at DESC); + +CREATE TABLE IF NOT EXISTS public.dlob_all_derived_ts ( + event_ts TIMESTAMPTZ NOT NULL, + id BIGSERIAL NOT NULL, + source TEXT NOT NULL, + market_type TEXT NOT NULL, + market_index INTEGER NOT NULL, + market_name TEXT NOT NULL, + is_indicative BOOLEAN NOT NULL DEFAULT FALSE, + ts_ms BIGINT NOT NULL, + slot BIGINT, + market_slot BIGINT, + mark_price NUMERIC, + oracle_price NUMERIC, + best_bid_price NUMERIC, + best_ask_price NUMERIC, + mid_price NUMERIC, + spread_quote NUMERIC, + spread_bps NUMERIC, + depth_levels INTEGER NOT NULL, + bid_levels INTEGER NOT NULL, + ask_levels INTEGER NOT NULL, + top_bid_size NUMERIC, + top_ask_size NUMERIC, + top_bid_notional NUMERIC, + top_ask_notional NUMERIC, + depth_bid_base NUMERIC NOT NULL DEFAULT 0, + depth_ask_base NUMERIC NOT NULL DEFAULT 0, + depth_bid_quote NUMERIC NOT NULL DEFAULT 0, + depth_ask_quote NUMERIC NOT NULL DEFAULT 0, + imbalance NUMERIC, + bids_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + asks_norm JSONB NOT NULL DEFAULT '[]'::jsonb, + raw_payload_hash TEXT NOT NULL, + inserted_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (event_ts, id) +); + +SELECT create_hypertable('dlob_all_derived_ts', 'event_ts', if_not_exists => TRUE, migrate_data => TRUE); + +CREATE UNIQUE INDEX IF NOT EXISTS dlob_all_derived_ts_dedupe_idx + ON public.dlob_all_derived_ts (event_ts, source, market_type, market_index, is_indicative, raw_payload_hash); + +CREATE INDEX IF NOT EXISTS dlob_all_derived_ts_market_ts_desc_idx + ON public.dlob_all_derived_ts (market_type, market_index, event_ts DESC); + +CREATE INDEX IF NOT EXISTS dlob_all_derived_ts_source_market_ts_desc_idx + ON public.dlob_all_derived_ts (source, market_type, market_index, event_ts DESC); + +DO $$ +BEGIN + PERFORM add_retention_policy('dlob_all_derived_ts', INTERVAL '30 days'); +EXCEPTION WHEN duplicate_object THEN NULL; +END $$; diff --git a/kustomize/base/kustomization.yaml b/kustomize/base/kustomization.yaml index de321a9..f5d8fbe 100644 --- a/kustomize/base/kustomization.yaml +++ b/kustomize/base/kustomization.yaml @@ -9,23 +9,10 @@ resources: - hasura/service.yaml - hasura/deployment.yaml - hasura/job-bootstrap.yaml - - api/service.yaml - - api/deployment.yaml - - ingestor/deployment.yaml - frontend/service.yaml - frontend/deployment.yaml - dlob/redis.yaml - dlob/publisher-deployment.yaml - - dlob/server-service.yaml - - dlob/server-deployment.yaml - - dlob-worker/deployment.yaml - - dlob-worker/deployment-drift.yaml - - dlob-depth-worker/deployment.yaml - - dlob-depth-worker/deployment-drift.yaml - - dlob-slippage-worker/deployment.yaml - - dlob-slippage-worker/deployment-drift.yaml - - dlob-ts-archiver/deployment.yaml - - dlob-ts-archiver/deployment-drift.yaml - candles-cache-worker/deployment.yaml configMapGenerator: @@ -35,27 +22,9 @@ configMapGenerator: - name: hasura-bootstrap-script files: - hasura/hasura-bootstrap.mjs - - name: dlob-worker-script - files: - - dlob-worker/worker.mjs - - name: dlob-depth-worker-script - files: - - dlob-depth-worker/worker.mjs - - name: dlob-slippage-worker-script - files: - - dlob-slippage-worker/worker.mjs - - name: dlob-ts-archiver-script - files: - - dlob-ts-archiver/worker.mjs - name: candles-cache-worker-script files: - candles-cache-worker/worker.mjs - - name: trade-api-wrapper - files: - - api/wrapper.mjs - - name: trade-api-upstream - files: - - api/server.mjs generatorOptions: disableNameSuffixHash: true diff --git a/kustomize/overlays/mevnode-bot/README.md b/kustomize/overlays/mevnode-bot/README.md index 921f7e6..fab16c4 100644 --- a/kustomize/overlays/mevnode-bot/README.md +++ b/kustomize/overlays/mevnode-bot/README.md @@ -6,15 +6,13 @@ Zakres: - `postgres` - `hasura` -- `trade-api` - `trade-frontend` (`NodePort 30081`) -- `trade-ingestor` - `dlob-redis` -- `dlob-publisher` -- `dlob-server` -- `dlob-worker` -- `dlob-depth-worker` -- `dlob-slippage-worker` +- `dlob-publisher-hot` +- `dlob-publisher-all` +- `dlob-hot-redis-to-postgres-raw-writer` +- `dlob-hot-postgres-to-postgres-derived-writer` +- `dlob-all-redis-to-postgres-derived-writer` Założenia runtime: @@ -26,11 +24,47 @@ Założenia runtime: - `trade-frontend` jest wystawiany przez `NodePort 30081`. - Namespace pozostaje `trade-staging`. +Topologia DLOB w tym overlayu: + +- `dlob-publisher-hot` to hot-path na wybrane markety (`PERP_MARKETS_TO_LOAD=0,20,72`, czyli `SOL-PERP`, `JTO-PERP`, `ADA-PERP`) i prefix Redis `DLOB_HOT` (`dlob-hot:`). +- `dlob-publisher-all` to szeroki feed do wszystkich marketów na prefix Redis `DLOB_ALL` (`dlob-all:`). +- `dlob-hot-redis-to-postgres-raw-writer` to schema-first writer: czyta `dlob-hot:` z Redis i zapisuje kanoniczny raw DLOB do PostgreSQL (`dlob_hot_snapshot_latest`, `dlob_hot_snapshot_ts`). +- `dlob-hot-postgres-to-postgres-derived-writer` czyta raw hot z PostgreSQL i buduje normalized layer (`dlob_hot_derived_latest`, `dlob_hot_derived_ts`). +- `LIVE` na `k3s`: sciezka `hot` jest juz aktywna jako `Redis -> raw PG -> derived PG`. +- `dlob-publisher-all` celowo nie ma patcha persistent store, żeby nie dublowac zapisow do Postgresa. +- `dlob-all-redis-to-postgres-derived-writer` zapisuje dla `all` tylko warstwe pochodna do PostgreSQL; bez `raw_ts`, zeby nie zjadac dysku w kilka dni. +- `LIVE` na `k3s`: sciezka `all` jest aktywna jako `Redis -> derived PG`. +- `all` zapisuje pelne `derived_ts` dla wszystkich marketow z `dlob-all:*`, bez samplingu i bez shortlisty marketow. + +Planowany zakres zapisu dla `all`: + +- tozsamosc i czas: `source`, `market_type`, `market_index`, `market_name`, `is_indicative`, `event_ts/ts_ms`, `slot`, `market_slot` +- cena i mikrostruktura: `mark_price`, `oracle_price`, `best_bid_price`, `best_ask_price`, `mid_price`, `spread_quote`, `spread_bps` +- plynnosc: `depth_levels`, `bid_levels`, `ask_levels`, `top_bid_size`, `top_ask_size`, `top_bid_notional`, `top_ask_notional`, `depth_bid_base`, `depth_ask_base`, `depth_bid_quote`, `depth_ask_quote`, `imbalance` +- drabinka do modelu/UI: `bids_norm`, `asks_norm` jako top `10` leveli z polami `price`, `sizeBase`, `notional`, `sources` +- kontrola jakosci i dedupe: `raw_payload_hash`, `updated_at` / `inserted_at` +- celowo pomijamy: pelny `raw payload`, `raw latest`, `raw ts`, `L3`, `best_makers` + +Polityka `hot` vs `all`: + +- `hot` i `all` rozniamy po roli operacyjnej, nie po samym mechanizmie subskrypcji. +- `hot` = `dlob-publisher-hot`: subset marketow, najnizsza latencja, najwyzszy priorytet operacyjny. +- `all` = `dlob-publisher-all`: pelne pokrycie marketow, feed do coverage/read-side/analityki, nizszy priorytet operacyjny niz `hot`. +- `USE_ORDER_SUBSCRIBER=true` moze byc wlaczone na obu publisherach; to nie definiuje roli `hot` vs `all`. +- `hot` powinien byc broniony jako pierwszy przy problemach z CPU/RAM/RPC i nie powinien byc degradowany przed `all`. +- Jesli trzeba ograniczac load albo robic fallback, najpierw degradujemy `all`, dopiero potem `hot`. + +Zalecana polityka runtime: + +- `hot`: utrzymywac `OrderSubscriber + gRPC` jako sciezke docelowa. +- `all`: domyslnie tez `OrderSubscriber + gRPC`, ale to pierwszy kandydat do przejscia na slabszy tryb przy presji zasobow. +- Kolejnosc degradacji dla `all`: `gRPC -> websocket -> polling`. +- Wylaczenie `USE_ORDER_SUBSCRIBER` traktowac jako tryb awaryjny/testowy, nie jako stan docelowy. + Wymagane sekrety: - `trade-postgres` - `trade-hasura` -- `trade-api` - `trade-frontend-tokens` - `trade-dlob-rpc` diff --git a/kustomize/overlays/mevnode-bot/dlob-all-redis-to-postgres-derived-writer.yaml b/kustomize/overlays/mevnode-bot/dlob-all-redis-to-postgres-derived-writer.yaml new file mode 100644 index 0000000..93dc58d --- /dev/null +++ b/kustomize/overlays/mevnode-bot/dlob-all-redis-to-postgres-derived-writer.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dlob-all-redis-to-postgres-derived-writer + annotations: + argocd.argoproj.io/sync-wave: "5" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: dlob-all-redis-to-postgres-derived-writer + template: + metadata: + labels: + app.kubernetes.io/name: dlob-all-redis-to-postgres-derived-writer + spec: + priorityClassName: dlob-all + imagePullSecrets: [] + containers: + - name: writer + image: gitea.mpabi.pl/trade/trade-dlob-server:all-derived-20260317-014016 + imagePullPolicy: IfNotPresent + env: + - name: DLOB_SOURCE + value: mevnode_bot_all_derived + - name: REDIS_HOST + value: dlob-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_KEY_PREFIX + value: "dlob-all:" + - name: DLOB_POLL_MS + value: "1000" + - name: NORMALIZED_DEPTH + value: "10" + - name: PRICE_PRECISION + value: "1000000" + - name: BASE_PRECISION + value: "1000000000" + - name: PGHOST + value: postgres + - name: PGPORT + value: "5432" + - name: PGUSER + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_PASSWORD + - name: PGDATABASE + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_DB + command: ["node", "/lib/scripts/dlobAllRedisToPostgresDerivedWriter.js"] + resources: + requests: + cpu: 250m + memory: 512Mi + limits: + cpu: 1000m + memory: 2Gi diff --git a/kustomize/overlays/mevnode-bot/dlob-hot-postgres-to-postgres-derived-writer.yaml b/kustomize/overlays/mevnode-bot/dlob-hot-postgres-to-postgres-derived-writer.yaml new file mode 100644 index 0000000..89cd7a7 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/dlob-hot-postgres-to-postgres-derived-writer.yaml @@ -0,0 +1,62 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dlob-hot-postgres-to-postgres-derived-writer + annotations: + argocd.argoproj.io/sync-wave: "5" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: dlob-hot-postgres-to-postgres-derived-writer + template: + metadata: + labels: + app.kubernetes.io/name: dlob-hot-postgres-to-postgres-derived-writer + spec: + priorityClassName: dlob-hot + imagePullSecrets: [] + containers: + - name: writer + image: gitea.mpabi.pl/trade/trade-dlob-server:hot-pg-events-20260320-205517 + imagePullPolicy: IfNotPresent + env: + - name: DLOB_SOURCE + value: mevnode_bot_hot_derived + - name: RAW_SOURCE + value: mevnode_bot_hot_raw + - name: DLOB_POLL_MS + value: "5000" + - name: NORMALIZED_DEPTH + value: "10" + - name: PRICE_PRECISION + value: "1000000" + - name: BASE_PRECISION + value: "1000000000" + - name: PGHOST + value: postgres + - name: PGPORT + value: "5432" + - name: PGUSER + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_PASSWORD + - name: PGDATABASE + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_DB + command: ["node", "/lib/scripts/dlobHotPostgresToPostgresDerivedWriter.js"] + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi diff --git a/kustomize/overlays/mevnode-bot/dlob-hot-redis-to-postgres-raw-writer.yaml b/kustomize/overlays/mevnode-bot/dlob-hot-redis-to-postgres-raw-writer.yaml new file mode 100644 index 0000000..36bc21f --- /dev/null +++ b/kustomize/overlays/mevnode-bot/dlob-hot-redis-to-postgres-raw-writer.yaml @@ -0,0 +1,60 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dlob-hot-redis-to-postgres-raw-writer + annotations: + argocd.argoproj.io/sync-wave: "5" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: dlob-hot-redis-to-postgres-raw-writer + template: + metadata: + labels: + app.kubernetes.io/name: dlob-hot-redis-to-postgres-raw-writer + spec: + priorityClassName: dlob-hot + imagePullSecrets: [] + containers: + - name: writer + image: gitea.mpabi.pl/trade/trade-dlob-server:hot-pg-events-20260320-205517 + imagePullPolicy: IfNotPresent + env: + - name: DLOB_SOURCE + value: mevnode_bot_hot_raw + - name: REDIS_HOST + value: dlob-redis + - name: REDIS_PORT + value: "6379" + - name: REDIS_KEY_PREFIX + value: "dlob-hot:" + - name: DLOB_POLL_MS + value: "5000" + - name: PGHOST + value: postgres + - name: PGPORT + value: "5432" + - name: PGUSER + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_USER + - name: PGPASSWORD + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_PASSWORD + - name: PGDATABASE + valueFrom: + secretKeyRef: + name: trade-postgres + key: POSTGRES_DB + command: ["node", "/lib/scripts/dlobHotRedisToPostgresRawWriter.js"] + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + cpu: 500m + memory: 1Gi diff --git a/kustomize/overlays/mevnode-bot/dlob-ingestor.mjs b/kustomize/overlays/mevnode-bot/dlob-ingestor.mjs deleted file mode 100644 index 8517237..0000000 --- a/kustomize/overlays/mevnode-bot/dlob-ingestor.mjs +++ /dev/null @@ -1,219 +0,0 @@ -import process from 'node:process'; -import { setTimeout as sleep } from 'node:timers/promises'; - -function getIsoNow() { - return new Date().toISOString(); -} - -function envString(name, fallback) { - const v = process.env[name]; - if (v == null) return fallback; - const s = String(v).trim(); - return s ? s : fallback; -} - -function envInt(name, fallback, { min, max } = {}) { - const v = process.env[name]; - if (v == null) return fallback; - const n = Number.parseInt(String(v), 10); - if (!Number.isFinite(n)) return fallback; - const low = typeof min === 'number' ? min : n; - const high = typeof max === 'number' ? max : n; - return Math.max(low, Math.min(high, n)); -} - -function envList(name, fallbackCsv) { - const raw = process.env[name] ?? fallbackCsv; - return String(raw) - .split(',') - .map((s) => s.trim()) - .filter(Boolean); -} - -function toIntOrNull(v) { - if (v == null) return null; - if (typeof v === 'number') return Number.isFinite(v) ? Math.trunc(v) : null; - if (typeof v === 'string') { - const s = v.trim(); - if (!s) return null; - const n = Number.parseInt(s, 10); - return Number.isFinite(n) ? n : null; - } - return null; -} - -function numStr(v) { - if (v == null) return null; - if (typeof v === 'number') return Number.isFinite(v) ? String(v) : null; - if (typeof v === 'string') { - const s = v.trim(); - return s ? s : null; - } - return null; -} - -function isoFromEpochMs(v) { - const n = typeof v === 'number' ? v : typeof v === 'string' ? Number(v.trim()) : NaN; - if (!Number.isFinite(n) || n <= 0) return null; - const d = new Date(n); - const ms = d.getTime(); - if (!Number.isFinite(ms)) return null; - return d.toISOString(); -} - -function resolveConfig() { - const hasuraUrl = envString('HASURA_GRAPHQL_URL', 'http://hasura:8080/v1/graphql'); - const hasuraAdminSecret = envString('HASURA_ADMIN_SECRET', ''); - if (!hasuraAdminSecret) throw new Error('Missing HASURA_ADMIN_SECRET'); - - const markets = envList('DLOB_MARKETS', 'SOL-PERP,PUMP-PERP'); - const pollMs = envInt('TICKS_POLL_MS', 1000, { min: 250, max: 60000 }); - const source = envString('TICKS_SOURCE', 'dlob_stats'); - - return { hasuraUrl, hasuraAdminSecret, markets, pollMs, source }; -} - -async function graphqlRequest(cfg, query, variables) { - const res = await fetch(cfg.hasuraUrl, { - method: 'POST', - headers: { - 'content-type': 'application/json', - 'x-hasura-admin-secret': cfg.hasuraAdminSecret, - }, - body: JSON.stringify({ query, variables }), - signal: AbortSignal.timeout(10000), - }); - - const text = await res.text(); - if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`); - - let json; - try { - json = JSON.parse(text); - } catch { - throw new Error(`Hasura: invalid json: ${text}`); - } - - if (json.errors?.length) throw new Error(json.errors.map((e) => e.message).join(' | ')); - return json.data; -} - -async function fetchStats(cfg) { - const query = ` - query DlobStatsLatest($markets: [String!]!) { - dlob_stats_latest(where: { market_name: { _in: $markets } }) { - market_name - market_index - ts - slot - oracle_price - mark_price - mid_price - best_bid_price - best_ask_price - updated_at - } - } - `; - - const data = await graphqlRequest(cfg, query, { markets: cfg.markets }); - const rows = Array.isArray(data?.dlob_stats_latest) ? data.dlob_stats_latest : []; - return rows; -} - -async function insertTicks(cfg, objects) { - if (!objects.length) return 0; - - const mutation = ` - mutation InsertTicks($objects: [drift_ticks_insert_input!]!) { - insert_drift_ticks(objects: $objects) { affected_rows } - } - `; - - const data = await graphqlRequest(cfg, mutation, { objects }); - return Number(data?.insert_drift_ticks?.affected_rows || 0); -} - -async function main() { - const cfg = resolveConfig(); - const lastUpdatedAtByMarket = new Map(); - - console.log( - JSON.stringify( - { - service: 'trade-ingestor', - mode: 'dlob_stats_ticks', - startedAt: getIsoNow(), - hasuraUrl: cfg.hasuraUrl, - markets: cfg.markets, - pollMs: cfg.pollMs, - source: cfg.source, - }, - null, - 2 - ) - ); - - while (true) { - try { - const rows = await fetchStats(cfg); - const nowIso = getIsoNow(); - - const objects = []; - for (const r of rows) { - const marketName = String(r?.market_name || '').trim(); - if (!marketName) continue; - - const updatedAt = r?.updated_at ? String(r.updated_at) : ''; - if (updatedAt && lastUpdatedAtByMarket.get(marketName) === updatedAt) continue; - if (updatedAt) lastUpdatedAtByMarket.set(marketName, updatedAt); - - const marketIndex = toIntOrNull(r?.market_index) ?? 0; - const dlobIso = isoFromEpochMs(r?.ts); - const tsIso = dlobIso || nowIso; - - const oraclePrice = numStr(r?.oracle_price) || numStr(r?.mark_price) || numStr(r?.mid_price); - const markPrice = numStr(r?.mark_price) || numStr(r?.mid_price) || oraclePrice; - if (!oraclePrice) continue; - - objects.push({ - ts: tsIso, - market_index: marketIndex, - symbol: marketName, - oracle_price: oraclePrice, - mark_price: markPrice, - oracle_slot: r?.slot == null ? null : String(r.slot), - source: cfg.source, - raw: { - from: 'dlob_stats_latest', - market_name: marketName, - market_index: marketIndex, - dlob: { - ts: r?.ts ?? null, - slot: r?.slot ?? null, - best_bid_price: r?.best_bid_price ?? null, - best_ask_price: r?.best_ask_price ?? null, - mid_price: r?.mid_price ?? null, - updated_at: updatedAt || null, - }, - }, - }); - } - - const inserted = await insertTicks(cfg, objects); - if (inserted) { - console.log(`[dlob-ticks] inserted=${inserted} ts=${nowIso}`); - } - } catch (err) { - console.error(`[dlob-ticks] error: ${String(err?.message || err)}`); - await sleep(2000); - } - - await sleep(cfg.pollMs); - } -} - -main().catch((err) => { - console.error(String(err?.stack || err)); - process.exitCode = 1; -}); diff --git a/kustomize/overlays/mevnode-bot/dlob-publisher-all-service.yaml b/kustomize/overlays/mevnode-bot/dlob-publisher-all-service.yaml new file mode 100644 index 0000000..73fd946 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/dlob-publisher-all-service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: dlob-publisher-all + annotations: + argocd.argoproj.io/sync-wave: "4" + prometheus.io/scrape: "true" + prometheus.io/port: "9464" + prometheus.io/path: /metrics +spec: + selector: + app.kubernetes.io/name: dlob-publisher-all + ports: + - name: http + port: 8080 + targetPort: 8080 + - name: metrics + port: 9464 + targetPort: 9464 diff --git a/kustomize/overlays/mevnode-bot/dlob-publisher-all.yaml b/kustomize/overlays/mevnode-bot/dlob-publisher-all.yaml new file mode 100644 index 0000000..641c5bb --- /dev/null +++ b/kustomize/overlays/mevnode-bot/dlob-publisher-all.yaml @@ -0,0 +1,102 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: dlob-publisher-all + annotations: + argocd.argoproj.io/sync-wave: "4" +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: dlob-publisher-all + template: + metadata: + labels: + app.kubernetes.io/name: dlob-publisher-all + spec: + imagePullSecrets: [] + containers: + - name: publisher + image: gitea.mpabi.pl/trade/trade-dlob-server:grpc-reconnect-20260319-023351 + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 + env: + - name: RUNNING_LOCAL + value: "true" + - name: LOCAL_CACHE + value: "true" + - name: ENV + value: mainnet-beta + - name: USE_WEBSOCKET + value: "true" + - name: USE_ORDER_SUBSCRIBER + value: "true" + - name: USE_GRPC + value: "true" + - name: DISABLE_GPA_REFRESH + value: "true" + - name: KILLSWITCH_SLOT_DIFF_THRESHOLD + value: "1000000000" + - name: ENABLE_TOB_MONITORING + value: "false" + - name: ELASTICACHE_HOST + value: dlob-redis + - name: ELASTICACHE_PORT + value: "6379" + - name: REDIS_CLIENT + value: DLOB_ALL + - name: FETCH_CONNECT_TIMEOUT_MS + value: "15000" + - name: FETCH_HEADERS_TIMEOUT_MS + value: "300000" + - name: FETCH_BODY_TIMEOUT_MS + value: "300000" + - name: GRPC_CLIENT + value: "yellowstone" + - name: ENDPOINT + valueFrom: + secretKeyRef: + name: trade-dlob-rpc + key: ENDPOINT + - name: WS_ENDPOINT + valueFrom: + secretKeyRef: + name: trade-dlob-rpc + key: WS_ENDPOINT + - name: GRPC_ENDPOINT + valueFrom: + secretKeyRef: + name: trade-dlob-rpc + key: GRPC_ENDPOINT + - name: TOKEN + valueFrom: + secretKeyRef: + name: trade-dlob-rpc + key: TOKEN + command: ["node", "/lib/publishers/dlobPublisher.js"] + startupProbe: + httpGet: + path: /startup + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 180 + readinessProbe: + httpGet: + path: /startup + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 24 + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 10 diff --git a/kustomize/overlays/mevnode-bot/dlob-redis-retention.yaml b/kustomize/overlays/mevnode-bot/dlob-redis-retention.yaml new file mode 100644 index 0000000..eba6448 --- /dev/null +++ b/kustomize/overlays/mevnode-bot/dlob-redis-retention.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: dlob-redis-retention +data: + clean.sh: | + #!/bin/sh + set -eu + now_s="$(date +%s)" + + for pattern in 'dlob-hot:*' 'dlob-all:*'; do + max_age_s=900 + if [ "$pattern" = 'dlob-all:*' ]; then + max_age_s=3600 + fi + + for key in $(redis-cli -h dlob-redis --scan --pattern "$pattern"); do + value="$(redis-cli -h dlob-redis --raw get "$key" || true)" + ts="$(printf '%s' "$value" | sed -n 's/.*"ts":\([0-9][0-9]*\).*/\1/p')" + if [ -z "$ts" ]; then + ts="$(printf '%s' "$value" | sed -n 's/.*"updatedAtTs":\([0-9][0-9]*\).*/\1/p')" + fi + [ -n "$ts" ] || continue + if [ $((now_s - ts / 1000)) -gt "$max_age_s" ]; then + redis-cli -h dlob-redis del "$key" >/dev/null + fi + done + done +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: dlob-redis-retention +spec: + schedule: "*/5 * * * *" + successfulJobsHistoryLimit: 1 + failedJobsHistoryLimit: 1 + jobTemplate: + spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: cleaner + image: redis:7-alpine + command: + - /bin/sh + - /scripts/clean.sh + resources: + requests: + cpu: 10m + memory: 32Mi + limits: + cpu: 100m + memory: 128Mi + volumeMounts: + - name: script + mountPath: /scripts + volumes: + - name: script + configMap: + name: dlob-redis-retention + defaultMode: 365 diff --git a/kustomize/overlays/mevnode-bot/kustomization.yaml b/kustomize/overlays/mevnode-bot/kustomization.yaml index 38987d0..668cbdf 100644 --- a/kustomize/overlays/mevnode-bot/kustomization.yaml +++ b/kustomize/overlays/mevnode-bot/kustomization.yaml @@ -12,30 +12,25 @@ resources: - ../../base/hasura/service.yaml - ../../base/hasura/deployment.yaml - ../../base/hasura/job-bootstrap.yaml - - ../../base/api/service.yaml - - ../../base/api/deployment.yaml - - ../../base/ingestor/deployment.yaml - ../../base/frontend/service.yaml - ../../base/frontend/deployment.yaml - ../../base/dlob/redis.yaml + - priority-classes.yaml - ../../base/dlob/publisher-deployment.yaml - - ../../base/dlob/server-service.yaml - - ../../base/dlob/server-deployment.yaml - - ../../base/dlob-worker/deployment.yaml - - ../../base/dlob-depth-worker/deployment.yaml - - ../../base/dlob-slippage-worker/deployment.yaml + - dlob-publisher-all.yaml + - dlob-publisher-all-service.yaml + - dlob-hot-redis-to-postgres-raw-writer.yaml + - dlob-hot-postgres-to-postgres-derived-writer.yaml + - dlob-all-redis-to-postgres-derived-writer.yaml + - dlob-redis-retention.yaml patchesStrategicMerge: - - patch-api.yaml - patch-frontend.yaml - patch-frontend-service.yaml - - patch-ingestor.yaml - patch-dlob-publisher.yaml - patch-dlob-publisher-storage.yaml - - patch-dlob-server.yaml - - patch-dlob-worker.yaml - - patch-dlob-depth-worker.yaml - - patch-dlob-slippage-worker.yaml + - patch-dlob-publisher-hot-priority.yaml + - patch-dlob-publisher-all-priority.yaml configMapGenerator: - name: postgres-initdb @@ -44,24 +39,6 @@ configMapGenerator: - name: hasura-bootstrap-script files: - ../../base/hasura/hasura-bootstrap.mjs - - name: trade-api-wrapper - files: - - ../../base/api/wrapper.mjs - - name: trade-api-upstream - files: - - ../../base/api/server.mjs - - name: dlob-worker-script - files: - - ../../base/dlob-worker/worker.mjs - - name: dlob-depth-worker-script - files: - - ../../base/dlob-depth-worker/worker.mjs - - name: dlob-slippage-worker-script - files: - - ../../base/dlob-slippage-worker/worker.mjs - - name: trade-dlob-ingestor-script - files: - - dlob-ingestor.mjs generatorOptions: disableNameSuffixHash: true diff --git a/kustomize/overlays/mevnode-bot/patch-dlob-depth-worker.yaml b/kustomize/overlays/mevnode-bot/patch-dlob-depth-worker.yaml deleted file mode 100644 index 32db728..0000000 --- a/kustomize/overlays/mevnode-bot/patch-dlob-depth-worker.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dlob-depth-worker -spec: - template: - spec: - containers: - - name: worker - env: - - name: DLOB_MARKETS - value: SOL-PERP,PUMP-PERP diff --git a/kustomize/overlays/mevnode-bot/patch-dlob-server.yaml b/kustomize/overlays/mevnode-bot/patch-dlob-publisher-all-priority.yaml similarity index 55% rename from kustomize/overlays/mevnode-bot/patch-dlob-server.yaml rename to kustomize/overlays/mevnode-bot/patch-dlob-publisher-all-priority.yaml index 1fae8d3..c7f865c 100644 --- a/kustomize/overlays/mevnode-bot/patch-dlob-server.yaml +++ b/kustomize/overlays/mevnode-bot/patch-dlob-publisher-all-priority.yaml @@ -1,8 +1,8 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: dlob-server + name: dlob-publisher-all spec: template: spec: - imagePullSecrets: [] + priorityClassName: dlob-all diff --git a/kustomize/overlays/mevnode-bot/patch-api.yaml b/kustomize/overlays/mevnode-bot/patch-dlob-publisher-hot-priority.yaml similarity index 55% rename from kustomize/overlays/mevnode-bot/patch-api.yaml rename to kustomize/overlays/mevnode-bot/patch-dlob-publisher-hot-priority.yaml index ced2120..e123d4c 100644 --- a/kustomize/overlays/mevnode-bot/patch-api.yaml +++ b/kustomize/overlays/mevnode-bot/patch-dlob-publisher-hot-priority.yaml @@ -1,8 +1,8 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: trade-api + name: dlob-publisher-hot spec: template: spec: - imagePullSecrets: [] + priorityClassName: dlob-hot diff --git a/kustomize/overlays/mevnode-bot/patch-dlob-publisher-storage.yaml b/kustomize/overlays/mevnode-bot/patch-dlob-publisher-storage.yaml index 75bf438..6b4bc29 100644 --- a/kustomize/overlays/mevnode-bot/patch-dlob-publisher-storage.yaml +++ b/kustomize/overlays/mevnode-bot/patch-dlob-publisher-storage.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: dlob-publisher + name: dlob-publisher-hot spec: replicas: 1 template: @@ -12,7 +12,7 @@ spec: - name: ENABLE_PERSISTENT_STORE value: "true" - name: DLOB_SOURCE - value: "mevnode_bot" + value: "mevnode_bot_hot" - name: PRICE_PRECISION value: "1000000" - name: BASE_PRECISION diff --git a/kustomize/overlays/mevnode-bot/patch-dlob-publisher.yaml b/kustomize/overlays/mevnode-bot/patch-dlob-publisher.yaml index 8bb97b8..29f643c 100644 --- a/kustomize/overlays/mevnode-bot/patch-dlob-publisher.yaml +++ b/kustomize/overlays/mevnode-bot/patch-dlob-publisher.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: dlob-publisher + name: dlob-publisher-hot spec: template: spec: @@ -10,13 +10,19 @@ spec: - name: publisher env: - name: PERP_MARKETS_TO_LOAD - value: "0,75" + value: "0,20,72" - name: USE_GRPC value: "true" - name: USE_WEBSOCKET value: "true" - name: DISABLE_GPA_REFRESH value: "true" + - name: FETCH_CONNECT_TIMEOUT_MS + value: "15000" + - name: FETCH_HEADERS_TIMEOUT_MS + value: "300000" + - name: FETCH_BODY_TIMEOUT_MS + value: "300000" - name: GRPC_CLIENT value: "yellowstone" - name: GRPC_ENDPOINT @@ -29,3 +35,27 @@ spec: secretKeyRef: name: trade-dlob-rpc key: TOKEN + startupProbe: + httpGet: + path: /startup + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 90 + readinessProbe: + httpGet: + path: /startup + port: http + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 3 + failureThreshold: 12 + livenessProbe: + httpGet: + path: /health + port: http + initialDelaySeconds: 30 + periodSeconds: 20 + timeoutSeconds: 3 + failureThreshold: 10 diff --git a/kustomize/overlays/mevnode-bot/patch-dlob-slippage-worker.yaml b/kustomize/overlays/mevnode-bot/patch-dlob-slippage-worker.yaml deleted file mode 100644 index f982f70..0000000 --- a/kustomize/overlays/mevnode-bot/patch-dlob-slippage-worker.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dlob-slippage-worker -spec: - template: - spec: - containers: - - name: worker - env: - - name: DLOB_MARKETS - value: SOL-PERP,PUMP-PERP diff --git a/kustomize/overlays/mevnode-bot/patch-dlob-worker.yaml b/kustomize/overlays/mevnode-bot/patch-dlob-worker.yaml deleted file mode 100644 index a6d7781..0000000 --- a/kustomize/overlays/mevnode-bot/patch-dlob-worker.yaml +++ /dev/null @@ -1,12 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: dlob-worker -spec: - template: - spec: - containers: - - name: worker - env: - - name: DLOB_MARKETS - value: SOL-PERP,PUMP-PERP diff --git a/kustomize/overlays/mevnode-bot/patch-ingestor.yaml b/kustomize/overlays/mevnode-bot/patch-ingestor.yaml deleted file mode 100644 index 9854ac8..0000000 --- a/kustomize/overlays/mevnode-bot/patch-ingestor.yaml +++ /dev/null @@ -1,39 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trade-ingestor -spec: - template: - spec: - imagePullSecrets: [] - containers: - - name: ingestor - image: node:20-slim - imagePullPolicy: IfNotPresent - env: - - name: HASURA_GRAPHQL_URL - value: http://hasura:8080/v1/graphql - - name: HASURA_ADMIN_SECRET - valueFrom: - secretKeyRef: - name: trade-hasura - key: HASURA_GRAPHQL_ADMIN_SECRET - - name: DLOB_MARKETS - value: SOL-PERP,PUMP-PERP - - name: TICKS_POLL_MS - value: "1000" - - name: TICKS_SOURCE - value: "dlob_stats" - command: - - node - args: - - /opt/dlob/dlob-ingestor.mjs - volumeMounts: - - name: dlob-script - mountPath: /opt/dlob/dlob-ingestor.mjs - subPath: dlob-ingestor.mjs - readOnly: true - volumes: - - name: dlob-script - configMap: - name: trade-dlob-ingestor-script diff --git a/kustomize/overlays/mevnode-bot/priority-classes.yaml b/kustomize/overlays/mevnode-bot/priority-classes.yaml new file mode 100644 index 0000000..3244edc --- /dev/null +++ b/kustomize/overlays/mevnode-bot/priority-classes.yaml @@ -0,0 +1,15 @@ +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: dlob-hot +value: 100000 +globalDefault: false +description: Higher scheduling priority for hot DLOB publisher. +--- +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: dlob-all +value: 50000 +globalDefault: false +description: Lower scheduling priority for all-market DLOB publisher. diff --git a/kustomize/overlays/prod/frontend-server.mjs b/kustomize/overlays/prod/frontend-server.mjs index 696b338..d110b2c 100644 --- a/kustomize/overlays/prod/frontend-server.mjs +++ b/kustomize/overlays/prod/frontend-server.mjs @@ -16,8 +16,6 @@ const STARTED_AT = new Date().toISOString(); const STATIC_DIR = process.env.STATIC_DIR || '/srv'; const BASIC_AUTH_FILE = process.env.BASIC_AUTH_FILE || '/tokens/frontend.json'; -const API_READ_TOKEN_FILE = process.env.API_READ_TOKEN_FILE || '/tokens/read.json'; -const API_UPSTREAM = process.env.API_UPSTREAM || process.env.API_URL || 'http://api:8787'; const HASURA_UPSTREAM = process.env.HASURA_UPSTREAM || 'http://hasura:8080'; const HASURA_GRAPHQL_PATH = process.env.HASURA_GRAPHQL_PATH || '/v1/graphql'; const GRAPHQL_CORS_ORIGIN = process.env.GRAPHQL_CORS_ORIGIN || process.env.CORS_ORIGIN || '*'; @@ -70,13 +68,6 @@ function loadBasicAuth() { return { username, password }; } -function loadApiReadToken() { - const j = readJson(API_READ_TOKEN_FILE); - const token = (j?.token || '').toString(); - if (!token) throw new Error(`Invalid API_READ_TOKEN_FILE: ${API_READ_TOKEN_FILE}`); - return token; -} - function send(res, status, headers, body) { res.statusCode = status; for (const [k, v] of Object.entries(headers || {})) res.setHeader(k, v); @@ -439,56 +430,6 @@ function readBody(req, limitBytes = 1024 * 16) { }); } -function proxyApi(req, res, apiReadToken) { - const upstreamBase = new URL(API_UPSTREAM); - const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`); - - const prefix = '/api'; - const strippedPath = inUrl.pathname === prefix ? '/' : inUrl.pathname.startsWith(prefix + '/') ? inUrl.pathname.slice(prefix.length) : null; - if (strippedPath == null) { - send(res, 404, { 'content-type': 'text/plain; charset=utf-8' }, 'not_found'); - return; - } - - const target = new URL(upstreamBase.toString()); - target.pathname = strippedPath || '/'; - target.search = inUrl.search; - - const isHttps = target.protocol === 'https:'; - const lib = isHttps ? https : http; - - const headers = stripHopByHopHeaders(req.headers); - delete headers.authorization; // basic auth from client must not leak upstream - headers.host = target.host; - headers.authorization = `Bearer ${apiReadToken}`; - - const upstreamReq = lib.request( - { - protocol: target.protocol, - hostname: target.hostname, - port: target.port || (isHttps ? 443 : 80), - method: req.method, - path: target.pathname + target.search, - headers, - }, - (upstreamRes) => { - const outHeaders = stripHopByHopHeaders(upstreamRes.headers); - res.writeHead(upstreamRes.statusCode || 502, outHeaders); - upstreamRes.pipe(res); - } - ); - - upstreamReq.on('error', (err) => { - if (!res.headersSent) { - send(res, 502, { 'content-type': 'text/plain; charset=utf-8' }, `bad_gateway: ${err?.message || err}`); - } else { - res.destroy(); - } - }); - - req.pipe(upstreamReq); -} - function withCors(res) { res.setHeader('access-control-allow-origin', GRAPHQL_CORS_ORIGIN); res.setHeader('access-control-allow-methods', 'GET,POST,OPTIONS'); @@ -734,26 +675,6 @@ async function handler(req, res) { } } - if (req.url?.startsWith('/api') && (req.url === '/api' || req.url.startsWith('/api/'))) { - if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') { - const user = resolveAuthenticatedUser(req); - if (!user) { - unauthorized(res); - return; - } - } - - let token; - try { - token = loadApiReadToken(); - } catch (e) { - send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e)); - return; - } - proxyApi(req, res, token); - return; - } - serveStatic(req, res); } @@ -797,11 +718,9 @@ server.listen(PORT, () => { service: 'trade-frontend', port: PORT, staticDir: STATIC_DIR, - apiUpstream: API_UPSTREAM, hasuraUpstream: HASURA_UPSTREAM, basicAuthFile: BASIC_AUTH_FILE, basicAuthMode: BASIC_AUTH_MODE, - apiReadTokenFile: API_READ_TOKEN_FILE, authUserHeader: AUTH_USER_HEADER, authMode: AUTH_MODE, htpasswdFile: HTPASSWD_FILE, diff --git a/kustomize/overlays/staging/dlob-ingestor.mjs b/kustomize/overlays/staging/dlob-ingestor.mjs deleted file mode 100644 index a25fed1..0000000 --- a/kustomize/overlays/staging/dlob-ingestor.mjs +++ /dev/null @@ -1,219 +0,0 @@ -import process from 'node:process'; -import { setTimeout as sleep } from 'node:timers/promises'; - -function getIsoNow() { - return new Date().toISOString(); -} - -function envString(name, fallback) { - const v = process.env[name]; - if (v == null) return fallback; - const s = String(v).trim(); - return s ? s : fallback; -} - -function envInt(name, fallback, { min, max } = {}) { - const v = process.env[name]; - if (v == null) return fallback; - const n = Number.parseInt(String(v), 10); - if (!Number.isFinite(n)) return fallback; - const low = typeof min === 'number' ? min : n; - const high = typeof max === 'number' ? max : n; - return Math.max(low, Math.min(high, n)); -} - -function envList(name, fallbackCsv) { - const raw = process.env[name] ?? fallbackCsv; - return String(raw) - .split(',') - .map((s) => s.trim()) - .filter(Boolean); -} - -function toIntOrNull(v) { - if (v == null) return null; - if (typeof v === 'number') return Number.isFinite(v) ? Math.trunc(v) : null; - if (typeof v === 'string') { - const s = v.trim(); - if (!s) return null; - const n = Number.parseInt(s, 10); - return Number.isFinite(n) ? n : null; - } - return null; -} - -function numStr(v) { - if (v == null) return null; - if (typeof v === 'number') return Number.isFinite(v) ? String(v) : null; - if (typeof v === 'string') { - const s = v.trim(); - return s ? s : null; - } - return null; -} - -function isoFromEpochMs(v) { - const n = typeof v === 'number' ? v : typeof v === 'string' ? Number(v.trim()) : NaN; - if (!Number.isFinite(n) || n <= 0) return null; - const d = new Date(n); - const ms = d.getTime(); - if (!Number.isFinite(ms)) return null; - return d.toISOString(); -} - -function resolveConfig() { - const hasuraUrl = envString('HASURA_GRAPHQL_URL', 'http://hasura:8080/v1/graphql'); - const hasuraAdminSecret = envString('HASURA_ADMIN_SECRET', ''); - if (!hasuraAdminSecret) throw new Error('Missing HASURA_ADMIN_SECRET'); - - const markets = envList('DLOB_MARKETS', 'SOL-PERP,DOGE-PERP,JUP-PERP'); - const pollMs = envInt('TICKS_POLL_MS', 1000, { min: 250, max: 60_000 }); - const source = envString('TICKS_SOURCE', 'dlob_stats'); - - return { hasuraUrl, hasuraAdminSecret, markets, pollMs, source }; -} - -async function graphqlRequest(cfg, query, variables) { - const res = await fetch(cfg.hasuraUrl, { - method: 'POST', - headers: { - 'content-type': 'application/json', - 'x-hasura-admin-secret': cfg.hasuraAdminSecret, - }, - body: JSON.stringify({ query, variables }), - signal: AbortSignal.timeout(10_000), - }); - - const text = await res.text(); - if (!res.ok) throw new Error(`Hasura HTTP ${res.status}: ${text}`); - - let json; - try { - json = JSON.parse(text); - } catch { - throw new Error(`Hasura: invalid json: ${text}`); - } - - if (json.errors?.length) throw new Error(json.errors.map((e) => e.message).join(' | ')); - return json.data; -} - -async function fetchStats(cfg) { - const query = ` - query DlobStatsLatest($markets: [String!]!) { - dlob_stats_latest(where: { market_name: { _in: $markets } }) { - market_name - market_index - ts - slot - oracle_price - mark_price - mid_price - best_bid_price - best_ask_price - updated_at - } - } - `; - - const data = await graphqlRequest(cfg, query, { markets: cfg.markets }); - const rows = Array.isArray(data?.dlob_stats_latest) ? data.dlob_stats_latest : []; - return rows; -} - -async function insertTicks(cfg, objects) { - if (!objects.length) return 0; - - const mutation = ` - mutation InsertTicks($objects: [drift_ticks_insert_input!]!) { - insert_drift_ticks(objects: $objects) { affected_rows } - } - `; - - const data = await graphqlRequest(cfg, mutation, { objects }); - return Number(data?.insert_drift_ticks?.affected_rows || 0); -} - -async function main() { - const cfg = resolveConfig(); - const lastUpdatedAtByMarket = new Map(); - - console.log( - JSON.stringify( - { - service: 'trade-ingestor', - mode: 'dlob_stats_ticks', - startedAt: getIsoNow(), - hasuraUrl: cfg.hasuraUrl, - markets: cfg.markets, - pollMs: cfg.pollMs, - source: cfg.source, - }, - null, - 2 - ) - ); - - while (true) { - try { - const rows = await fetchStats(cfg); - const nowIso = getIsoNow(); - - const objects = []; - for (const r of rows) { - const marketName = String(r?.market_name || '').trim(); - if (!marketName) continue; - - const updatedAt = r?.updated_at ? String(r.updated_at) : ''; - if (updatedAt && lastUpdatedAtByMarket.get(marketName) === updatedAt) continue; - if (updatedAt) lastUpdatedAtByMarket.set(marketName, updatedAt); - - const marketIndex = toIntOrNull(r?.market_index) ?? 0; - const dlobIso = isoFromEpochMs(r?.ts); - const tsIso = dlobIso || nowIso; - - const oraclePrice = numStr(r?.oracle_price) || numStr(r?.mark_price) || numStr(r?.mid_price); - const markPrice = numStr(r?.mark_price) || numStr(r?.mid_price) || oraclePrice; - if (!oraclePrice) continue; - - objects.push({ - ts: tsIso, - market_index: marketIndex, - symbol: marketName, - oracle_price: oraclePrice, - mark_price: markPrice, - oracle_slot: r?.slot == null ? null : String(r.slot), - source: cfg.source, - raw: { - from: 'dlob_stats_latest', - market_name: marketName, - market_index: marketIndex, - dlob: { - ts: r?.ts ?? null, - slot: r?.slot ?? null, - best_bid_price: r?.best_bid_price ?? null, - best_ask_price: r?.best_ask_price ?? null, - mid_price: r?.mid_price ?? null, - updated_at: updatedAt || null, - }, - }, - }); - } - - const inserted = await insertTicks(cfg, objects); - if (inserted) { - console.log(`[dlob-ticks] inserted=${inserted} ts=${nowIso}`); - } - } catch (err) { - console.error(`[dlob-ticks] error: ${String(err?.message || err)}`); - await sleep(2_000); - } - - await sleep(cfg.pollMs); - } -} - -main().catch((err) => { - console.error(String(err?.stack || err)); - process.exitCode = 1; -}); diff --git a/kustomize/overlays/staging/fake-ingestor.mjs b/kustomize/overlays/staging/fake-ingestor.mjs deleted file mode 100644 index feea705..0000000 --- a/kustomize/overlays/staging/fake-ingestor.mjs +++ /dev/null @@ -1,487 +0,0 @@ -import fs from 'node:fs'; - -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -function envString(name, fallback) { - const v = process.env[name]; - if (v == null) return fallback; - const s = String(v).trim(); - return s ? s : fallback; -} - -function envNumber(name, fallback) { - const v = process.env[name]; - if (v == null) return fallback; - const n = Number(v); - return Number.isFinite(n) ? n : fallback; -} - -function envInt(name, fallback, { min, max } = {}) { - const v = process.env[name]; - if (v == null) return fallback; - const n = Number.parseInt(String(v), 10); - if (!Number.isInteger(n)) return fallback; - const nn = Math.max(min ?? n, Math.min(max ?? n, n)); - return nn; -} - -function readJson(filePath) { - try { - const raw = fs.readFileSync(filePath, 'utf8'); - return JSON.parse(raw); - } catch { - return undefined; - } -} - -function readTokenFromFile(filePath) { - const json = readJson(filePath); - const raw = json?.token || json?.jwt || json?.authToken; - const tok = typeof raw === 'string' ? raw.trim() : ''; - return tok ? tok : undefined; -} - -function urlWithPath(baseUrl, path) { - const u = new URL(baseUrl); - const basePath = u.pathname && u.pathname !== '/' ? u.pathname.replace(/\/$/, '') : ''; - u.pathname = `${basePath}${path}`; - return u; -} - -async function httpJson(url, options) { - const res = await fetch(url, options); - const text = await res.text(); - let json; - try { - json = JSON.parse(text); - } catch { - json = undefined; - } - return { ok: res.ok, status: res.status, json, text }; -} - -function mean(values) { - if (!values.length) return 0; - return values.reduce((a, b) => a + b, 0) / values.length; -} - -function stddev(values) { - if (!values.length) return 0; - const m = mean(values); - const v = values.reduce((acc, x) => acc + (x - m) * (x - m), 0) / values.length; - return Math.sqrt(v); -} - -function quantile(values, q) { - if (!values.length) return 0; - const sorted = values.slice().sort((a, b) => a - b); - const pos = (sorted.length - 1) * q; - const base = Math.floor(pos); - const rest = pos - base; - if (sorted[base + 1] == null) return sorted[base]; - return sorted[base] + rest * (sorted[base + 1] - sorted[base]); -} - -function randn() { - // Box–Muller - let u = 0; - let v = 0; - while (u === 0) u = Math.random(); - while (v === 0) v = Math.random(); - return Math.sqrt(-2 * Math.log(u)) * Math.cos(2 * Math.PI * v); -} - -function clamp(v, min, max) { - if (v < min) return min; - if (v > max) return max; - return v; -} - -function isTruthy(value) { - const v = String(value ?? '') - .trim() - .toLowerCase(); - if (!v) return false; - return !['0', 'false', 'off', 'no', 'none', 'disabled'].includes(v); -} - -function parsePriceFromTick(t) { - const mp = t?.mark_price ?? t?.markPrice; - const op = t?.oracle_price ?? t?.oraclePrice ?? t?.price; - const v = mp != null ? Number(mp) : Number(op); - return Number.isFinite(v) && v > 0 ? v : null; -} - -function computeLogReturns(prices) { - const out = []; - for (let i = 1; i < prices.length; i++) { - const a = prices[i - 1]; - const b = prices[i]; - if (!(a > 0) || !(b > 0)) continue; - const r = Math.log(b / a); - if (Number.isFinite(r)) out.push(r); - } - return out; -} - -class BlockSampler { - #values; - #minBlock; - #maxBlock; - #idx = 0; - #end = 0; - #remaining = 0; - - constructor(values, { minBlock, maxBlock }) { - this.#values = values.slice(); - this.#minBlock = Math.max(1, minBlock); - this.#maxBlock = Math.max(this.#minBlock, maxBlock); - this.#startNewBlock(); - } - - #startNewBlock() { - const n = this.#values.length; - if (n <= 1) { - this.#idx = 0; - this.#end = n; - this.#remaining = n; - return; - } - - const start = Math.floor(Math.random() * n); - const span = this.#maxBlock - this.#minBlock + 1; - const len = this.#minBlock + Math.floor(Math.random() * span); - - this.#idx = start; - this.#end = Math.min(n, start + len); - this.#remaining = this.#end - this.#idx; - } - - next() { - if (this.#values.length === 0) return 0; - if (this.#remaining <= 0 || this.#idx >= this.#end) this.#startNewBlock(); - - const v = this.#values[this.#idx]; - this.#idx += 1; - this.#remaining -= 1; - - return Number.isFinite(v) ? v : 0; - } -} - -async function fetchSeedTicksPage({ apiBase, readToken, symbol, source, limit, to }) { - const u = urlWithPath(apiBase, '/v1/ticks'); - u.searchParams.set('symbol', symbol); - u.searchParams.set('limit', String(limit)); - if (source) u.searchParams.set('source', source); - if (to) u.searchParams.set('to', to); - - const res = await httpJson(u.toString(), { - method: 'GET', - headers: { - authorization: `Bearer ${readToken}`, - }, - }); - - if (!res.ok) throw new Error(`seed_ticks_http_${res.status}: ${res.text}`); - if (!res.json?.ok) throw new Error(`seed_ticks_error: ${res.json?.error || res.text}`); - - const ticks = Array.isArray(res.json?.ticks) ? res.json.ticks : []; - return ticks; -} - -async function fetchSeedTicksPaged({ apiBase, readToken, symbol, source, desiredLimit, pageLimit, maxPages }) { - const pages = []; - let cursorTo = null; - - for (let page = 0; page < maxPages; page++) { - const remaining = desiredLimit - pages.reduce((acc, p) => acc + p.length, 0); - if (remaining <= 0) break; - - const limit = Math.min(pageLimit, remaining); - const ticks = await fetchSeedTicksPage({ apiBase, readToken, symbol, source, limit, to: cursorTo }); - if (!ticks.length) break; - - // Server returns ascending ticks; we unshift to keep overall chronological order. - pages.unshift(ticks); - - const oldestTs = String(ticks[0]?.ts || '').trim(); - if (!oldestTs) break; - const oldestMs = Date.parse(oldestTs); - if (!Number.isFinite(oldestMs)) break; - cursorTo = new Date(oldestMs - 1).toISOString(); - } - - const flat = pages.flat(); - if (!flat.length) return flat; - - // Best-effort de-duplication (in case of overlapping `to` bounds). - const seen = new Set(); - const out = []; - for (const t of flat) { - const ts = String(t?.ts || ''); - const key = ts ? ts : JSON.stringify(t); - if (seen.has(key)) continue; - seen.add(key); - out.push(t); - } - return out; -} - -async function ingestTick({ apiBase, writeToken, tick }) { - const u = urlWithPath(apiBase, '/v1/ingest/tick'); - const res = await httpJson(u.toString(), { - method: 'POST', - headers: { - 'content-type': 'application/json', - authorization: `Bearer ${writeToken}`, - }, - body: JSON.stringify(tick), - }); - - if (!res.ok) throw new Error(`ingest_http_${res.status}: ${res.text}`); - if (!res.json?.ok) throw new Error(`ingest_error: ${res.json?.error || res.text}`); - return res.json?.id || null; -} - -function formatNumeric(value) { - if (!Number.isFinite(value)) return '0'; - // Avoid scientific notation for small values while keeping reasonable precision. - const abs = Math.abs(value); - if (abs === 0) return '0'; - if (abs >= 1) return value.toFixed(8).replace(/\.?0+$/, ''); - if (abs >= 0.01) return value.toFixed(10).replace(/\.?0+$/, ''); - if (abs >= 0.0001) return value.toFixed(12).replace(/\.?0+$/, ''); - return value.toFixed(16).replace(/\.?0+$/, ''); -} - -async function main() { - const apiBase = envString('INGEST_API_URL', 'http://trade-api:8787'); - const symbol = envString('MARKET_NAME', envString('SYMBOL', 'PUMP-PERP')); - const source = envString('SOURCE', 'drift_oracle'); - const intervalMs = envInt('INTERVAL_MS', 1000, { min: 50, max: 60_000 }); - - const readTokenFile = envString('FAKE_READ_TOKEN_FILE', envString('READ_TOKEN_FILE', '/tokens/read.json')); - const writeTokenFile = envString('FAKE_WRITE_TOKEN_FILE', envString('WRITE_TOKEN_FILE', '/app/tokens/alg.json')); - - const seedLimitDesired = envInt('FAKE_SEED_LIMIT', 50_000, { min: 50, max: 200_000 }); - const seedPageLimit = envInt('FAKE_SEED_PAGE_LIMIT', 5000, { min: 50, max: 5000 }); - const seedMaxPages = envInt( - 'FAKE_SEED_MAX_PAGES', - Math.ceil(seedLimitDesired / seedPageLimit) + 2, - { min: 1, max: 200 } - ); - const seedSourceRaw = process.env.FAKE_SEED_SOURCE; - const seedSource = seedSourceRaw == null ? source : String(seedSourceRaw).trim(); - - const minBlock = envInt('FAKE_BLOCK_MIN', 120, { min: 1, max: 50_000 }); - const maxBlock = envInt('FAKE_BLOCK_MAX', 1200, { min: 1, max: 50_000 }); - - const volScale = envNumber('FAKE_VOL_SCALE', 1); - const noiseScale = envNumber('FAKE_NOISE_SCALE', 0.05); - const meanReversion = envNumber('FAKE_MEAN_REVERSION', 0.0002); - const markNoiseBps = envNumber('FAKE_MARK_NOISE_BPS', 5); - const logEvery = envInt('FAKE_LOG_EVERY', 30, { min: 1, max: 10_000 }); - - const marketIndexEnv = process.env.MARKET_INDEX; - const marketIndexFallback = Number.isInteger(Number(marketIndexEnv)) ? Number(marketIndexEnv) : 0; - const startPriceEnv = envNumber('FAKE_START_PRICE', 0); - - const clampEnabled = isTruthy(process.env.FAKE_CLAMP ?? '1'); - const clampQLow = clamp(envNumber('FAKE_CLAMP_Q_LOW', 0.05), 0.0, 0.49); - const clampQHigh = clamp(envNumber('FAKE_CLAMP_Q_HIGH', 0.95), 0.51, 1.0); - const clampLowMult = envNumber('FAKE_CLAMP_LOW_MULT', 0.8); - const clampHighMult = envNumber('FAKE_CLAMP_HIGH_MULT', 1.2); - - const readToken = readTokenFromFile(readTokenFile); - const writeToken = readTokenFromFile(writeTokenFile); - if (!writeToken) throw new Error(`Missing write token (expected JSON token at ${writeTokenFile})`); - - let seedTicks = []; - let seedPrices = []; - let marketIndex = marketIndexFallback; - let seedFromTs = null; - let seedToTs = null; - - if (!readToken) { - console.warn(`[fake-ingestor] No read token at ${readTokenFile}; running without seed data.`); - } else { - seedTicks = await fetchSeedTicksPaged({ - apiBase, - readToken, - symbol, - source: seedSource ? seedSource : undefined, - desiredLimit: seedLimitDesired, - pageLimit: seedPageLimit, - maxPages: seedMaxPages, - }); - - seedFromTs = seedTicks.length ? String(seedTicks[0]?.ts || '') : null; - seedToTs = seedTicks.length ? String(seedTicks[seedTicks.length - 1]?.ts || '') : null; - - for (const t of seedTicks) { - const p = parsePriceFromTick(t); - if (p != null) seedPrices.push(p); - } - - const lastTick = seedTicks.length ? seedTicks[seedTicks.length - 1] : null; - const mi = lastTick?.market_index ?? lastTick?.marketIndex; - if (Number.isInteger(Number(mi))) marketIndex = Number(mi); - } - - if (!seedPrices.length && startPriceEnv > 0) { - seedPrices = [startPriceEnv]; - } - if (!seedPrices.length) { - throw new Error( - 'No seed prices available. Provide FAKE_START_PRICE (and optionally MARKET_INDEX) or mount a read token to seed from /v1/ticks.' - ); - } - - const returnsRaw = computeLogReturns(seedPrices); - const mu = mean(returnsRaw); - const sigma = stddev(returnsRaw); - const center = envString('FAKE_CENTER_RETURNS', '1') !== '0'; - const returns = returnsRaw.length - ? center - ? returnsRaw.map((r) => r - mu) - : returnsRaw - : []; - - const sampler = new BlockSampler(returns.length ? returns : [0], { minBlock, maxBlock }); - - const pLow = quantile(seedPrices, clampQLow); - const p50 = quantile(seedPrices, 0.5); - const pHigh = quantile(seedPrices, clampQHigh); - const clampMin = pLow > 0 ? pLow * clampLowMult : Math.min(...seedPrices) * clampLowMult; - const clampMax = pHigh > 0 ? pHigh * clampHighMult : Math.max(...seedPrices) * clampHighMult; - - let logPrice = Math.log(seedPrices[seedPrices.length - 1]); - const targetLog = Math.log(p50 > 0 ? p50 : seedPrices[seedPrices.length - 1]); - - console.log( - JSON.stringify( - { - service: 'trade-fake-ingestor', - apiBase, - symbol, - source, - intervalMs, - marketIndex, - seed: { - ok: Boolean(seedTicks.length), - desiredLimit: seedLimitDesired, - pageLimit: seedPageLimit, - maxPages: seedMaxPages, - source: seedSource || null, - ticks: seedTicks.length, - prices: seedPrices.length, - fromTs: seedFromTs, - toTs: seedToTs, - }, - model: { - type: 'block-bootstrap-returns', - centerReturns: center, - mu, - sigma, - volScale, - noiseScale, - meanReversion, - block: { minBlock, maxBlock }, - clamp: clampEnabled ? { min: clampMin, max: clampMax } : { disabled: true }, - }, - }, - null, - 2 - ) - ); - - let stopping = false; - process.on('SIGINT', () => { - stopping = true; - }); - process.on('SIGTERM', () => { - stopping = true; - }); - - let tickCount = 0; - let nextAt = Date.now(); - - while (!stopping) { - const now = Date.now(); - if (now < nextAt) await sleep(nextAt - now); - nextAt += intervalMs; - - const r = sampler.next(); - const noise = (Number.isFinite(sigma) ? sigma : 0) * noiseScale * randn(); - const revert = Number.isFinite(meanReversion) ? meanReversion * (targetLog - logPrice) : 0; - const step = (Number.isFinite(r) ? r : 0) * volScale + noise + revert; - - logPrice += step; - let oraclePrice = Math.exp(logPrice); - if (clampEnabled && Number.isFinite(clampMin) && Number.isFinite(clampMax) && clampMax > clampMin) { - oraclePrice = clamp(oraclePrice, clampMin, clampMax); - logPrice = Math.log(oraclePrice); - } - - const markNoise = (markNoiseBps / 10_000) * randn(); - const markPrice = oraclePrice * (1 + markNoise); - - const ts = new Date().toISOString(); - const tick = { - ts, - market_index: marketIndex, - symbol, - oracle_price: formatNumeric(oraclePrice), - mark_price: formatNumeric(markPrice), - source, - raw: { - fake: true, - model: 'block-bootstrap-returns', - seeded: seedTicks.length - ? { - symbol, - source: seedSource || null, - desiredLimit: seedLimitDesired, - pageLimit: seedPageLimit, - maxPages: seedMaxPages, - fromTs: seedFromTs, - toTs: seedToTs, - } - : { - symbol, - source: null, - desiredLimit: 0, - pageLimit: seedPageLimit, - maxPages: seedMaxPages, - fromTs: null, - toTs: null, - }, - }, - }; - - try { - await ingestTick({ apiBase, writeToken, tick }); - tickCount += 1; - if (tickCount % logEvery === 0) { - console.log( - `[fake-ingestor] ok ticks=${tickCount} ts=${ts} px=${tick.oracle_price} mark=${tick.mark_price} clamp=[${formatNumeric( - clampMin - )},${formatNumeric(clampMax)}]` - ); - } - } catch (err) { - console.warn(`[fake-ingestor] ingest failed: ${String(err?.message || err)}`); - await sleep(Math.min(5000, Math.max(250, intervalMs))); - } - } - - console.log('[fake-ingestor] stopped'); -} - -main().catch((err) => { - console.error(String(err?.stack || err)); - process.exitCode = 1; -}); diff --git a/kustomize/overlays/staging/frontend-server.mjs b/kustomize/overlays/staging/frontend-server.mjs index 696b338..d110b2c 100644 --- a/kustomize/overlays/staging/frontend-server.mjs +++ b/kustomize/overlays/staging/frontend-server.mjs @@ -16,8 +16,6 @@ const STARTED_AT = new Date().toISOString(); const STATIC_DIR = process.env.STATIC_DIR || '/srv'; const BASIC_AUTH_FILE = process.env.BASIC_AUTH_FILE || '/tokens/frontend.json'; -const API_READ_TOKEN_FILE = process.env.API_READ_TOKEN_FILE || '/tokens/read.json'; -const API_UPSTREAM = process.env.API_UPSTREAM || process.env.API_URL || 'http://api:8787'; const HASURA_UPSTREAM = process.env.HASURA_UPSTREAM || 'http://hasura:8080'; const HASURA_GRAPHQL_PATH = process.env.HASURA_GRAPHQL_PATH || '/v1/graphql'; const GRAPHQL_CORS_ORIGIN = process.env.GRAPHQL_CORS_ORIGIN || process.env.CORS_ORIGIN || '*'; @@ -70,13 +68,6 @@ function loadBasicAuth() { return { username, password }; } -function loadApiReadToken() { - const j = readJson(API_READ_TOKEN_FILE); - const token = (j?.token || '').toString(); - if (!token) throw new Error(`Invalid API_READ_TOKEN_FILE: ${API_READ_TOKEN_FILE}`); - return token; -} - function send(res, status, headers, body) { res.statusCode = status; for (const [k, v] of Object.entries(headers || {})) res.setHeader(k, v); @@ -439,56 +430,6 @@ function readBody(req, limitBytes = 1024 * 16) { }); } -function proxyApi(req, res, apiReadToken) { - const upstreamBase = new URL(API_UPSTREAM); - const inUrl = new URL(req.url || '/', `http://${req.headers.host || 'localhost'}`); - - const prefix = '/api'; - const strippedPath = inUrl.pathname === prefix ? '/' : inUrl.pathname.startsWith(prefix + '/') ? inUrl.pathname.slice(prefix.length) : null; - if (strippedPath == null) { - send(res, 404, { 'content-type': 'text/plain; charset=utf-8' }, 'not_found'); - return; - } - - const target = new URL(upstreamBase.toString()); - target.pathname = strippedPath || '/'; - target.search = inUrl.search; - - const isHttps = target.protocol === 'https:'; - const lib = isHttps ? https : http; - - const headers = stripHopByHopHeaders(req.headers); - delete headers.authorization; // basic auth from client must not leak upstream - headers.host = target.host; - headers.authorization = `Bearer ${apiReadToken}`; - - const upstreamReq = lib.request( - { - protocol: target.protocol, - hostname: target.hostname, - port: target.port || (isHttps ? 443 : 80), - method: req.method, - path: target.pathname + target.search, - headers, - }, - (upstreamRes) => { - const outHeaders = stripHopByHopHeaders(upstreamRes.headers); - res.writeHead(upstreamRes.statusCode || 502, outHeaders); - upstreamRes.pipe(res); - } - ); - - upstreamReq.on('error', (err) => { - if (!res.headersSent) { - send(res, 502, { 'content-type': 'text/plain; charset=utf-8' }, `bad_gateway: ${err?.message || err}`); - } else { - res.destroy(); - } - }); - - req.pipe(upstreamReq); -} - function withCors(res) { res.setHeader('access-control-allow-origin', GRAPHQL_CORS_ORIGIN); res.setHeader('access-control-allow-methods', 'GET,POST,OPTIONS'); @@ -734,26 +675,6 @@ async function handler(req, res) { } } - if (req.url?.startsWith('/api') && (req.url === '/api' || req.url.startsWith('/api/'))) { - if (AUTH_MODE !== 'off' && AUTH_MODE !== 'none' && AUTH_MODE !== 'disabled') { - const user = resolveAuthenticatedUser(req); - if (!user) { - unauthorized(res); - return; - } - } - - let token; - try { - token = loadApiReadToken(); - } catch (e) { - send(res, 500, { 'content-type': 'text/plain; charset=utf-8' }, String(e?.message || e)); - return; - } - proxyApi(req, res, token); - return; - } - serveStatic(req, res); } @@ -797,11 +718,9 @@ server.listen(PORT, () => { service: 'trade-frontend', port: PORT, staticDir: STATIC_DIR, - apiUpstream: API_UPSTREAM, hasuraUpstream: HASURA_UPSTREAM, basicAuthFile: BASIC_AUTH_FILE, basicAuthMode: BASIC_AUTH_MODE, - apiReadTokenFile: API_READ_TOKEN_FILE, authUserHeader: AUTH_USER_HEADER, authMode: AUTH_MODE, htpasswdFile: HTPASSWD_FILE, diff --git a/kustomize/overlays/staging/ingestor-dlob-patch.yaml b/kustomize/overlays/staging/ingestor-dlob-patch.yaml deleted file mode 100644 index f5dc313..0000000 --- a/kustomize/overlays/staging/ingestor-dlob-patch.yaml +++ /dev/null @@ -1,36 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trade-ingestor -spec: - template: - spec: - containers: - - name: ingestor - image: node:20-slim - imagePullPolicy: IfNotPresent - env: - - name: HASURA_GRAPHQL_URL - value: http://hasura:8080/v1/graphql - - name: HASURA_ADMIN_SECRET - valueFrom: - secretKeyRef: - name: trade-hasura - key: HASURA_GRAPHQL_ADMIN_SECRET - - name: DLOB_MARKETS - value: SOL-PERP,DOGE-PERP,JUP-PERP - - name: TICKS_POLL_MS - value: "1000" - - name: TICKS_SOURCE - value: "dlob_stats" - command: ["node"] - args: ["/opt/dlob/dlob-ingestor.mjs"] - volumeMounts: - - name: dlob-script - mountPath: /opt/dlob/dlob-ingestor.mjs - subPath: dlob-ingestor.mjs - readOnly: true - volumes: - - name: dlob-script - configMap: - name: trade-dlob-ingestor-script diff --git a/kustomize/overlays/staging/ingestor-fake-patch.yaml b/kustomize/overlays/staging/ingestor-fake-patch.yaml deleted file mode 100644 index f601924..0000000 --- a/kustomize/overlays/staging/ingestor-fake-patch.yaml +++ /dev/null @@ -1,54 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: trade-ingestor -spec: - template: - spec: - containers: - - name: ingestor - env: - - name: FAKE_READ_TOKEN_FILE - value: "/tokens/read.json" - - name: FAKE_WRITE_TOKEN_FILE - value: "/app/tokens/alg.json" - - name: FAKE_SEED_LIMIT - value: "50000" - - name: FAKE_SEED_PAGE_LIMIT - value: "5000" - - name: FAKE_SEED_MAX_PAGES - value: "20" - - name: FAKE_BLOCK_MIN - value: "300" - - name: FAKE_BLOCK_MAX - value: "1800" - - name: FAKE_VOL_SCALE - value: "1.8" - - name: FAKE_NOISE_SCALE - value: "0.03" - - name: FAKE_MEAN_REVERSION - value: "0.0001" - - name: FAKE_CLAMP_Q_LOW - value: "0.02" - - name: FAKE_CLAMP_Q_HIGH - value: "0.98" - - name: FAKE_CLAMP_LOW_MULT - value: "0.7" - - name: FAKE_CLAMP_HIGH_MULT - value: "1.3" - command: ["node"] - args: ["/opt/fake/fake-ingestor.mjs"] - volumeMounts: - - name: fake-script - mountPath: /opt/fake - readOnly: true - - name: read-tokens - mountPath: /tokens - readOnly: true - volumes: - - name: fake-script - configMap: - name: trade-fake-ingestor-script - - name: read-tokens - secret: - secretName: trade-frontend-tokens diff --git a/kustomize/overlays/staging/kustomization.yaml b/kustomize/overlays/staging/kustomization.yaml index 7d11915..b37f104 100644 --- a/kustomize/overlays/staging/kustomization.yaml +++ b/kustomize/overlays/staging/kustomization.yaml @@ -13,12 +13,8 @@ patchesStrategicMerge: - hasura-patch.yaml - frontend-auth-patch.yaml - frontend-graphql-proxy-patch.yaml - - ingestor-dlob-patch.yaml configMapGenerator: - - name: trade-dlob-ingestor-script - files: - - dlob-ingestor.mjs - name: trade-frontend-server-script files: - frontend-server.mjs