diff --git a/kustomize/overlays/staging/fake-ingestor.mjs b/kustomize/overlays/staging/fake-ingestor.mjs index 50c4b9e..feea705 100644 --- a/kustomize/overlays/staging/fake-ingestor.mjs +++ b/kustomize/overlays/staging/fake-ingestor.mjs @@ -99,6 +99,14 @@ function clamp(v, min, max) { return v; } +function isTruthy(value) { + const v = String(value ?? '') + .trim() + .toLowerCase(); + if (!v) return false; + return !['0', 'false', 'off', 'no', 'none', 'disabled'].includes(v); +} + function parsePriceFromTick(t) { const mp = t?.mark_price ?? t?.markPrice; const op = t?.oracle_price ?? t?.oraclePrice ?? t?.price; @@ -123,6 +131,7 @@ class BlockSampler { #minBlock; #maxBlock; #idx = 0; + #end = 0; #remaining = 0; constructor(values, { minBlock, maxBlock }) { @@ -136,7 +145,8 @@ class BlockSampler { const n = this.#values.length; if (n <= 1) { this.#idx = 0; - this.#remaining = 1; + this.#end = n; + this.#remaining = n; return; } @@ -145,28 +155,28 @@ class BlockSampler { const len = this.#minBlock + Math.floor(Math.random() * span); this.#idx = start; - this.#remaining = Math.min(len, n); + this.#end = Math.min(n, start + len); + this.#remaining = this.#end - this.#idx; } next() { - const n = this.#values.length; - if (n === 0) return 0; + if (this.#values.length === 0) return 0; + if (this.#remaining <= 0 || this.#idx >= this.#end) this.#startNewBlock(); - if (this.#remaining <= 0) this.#startNewBlock(); const v = this.#values[this.#idx]; - - this.#idx = (this.#idx + 1) % n; + this.#idx += 1; this.#remaining -= 1; return Number.isFinite(v) ? v : 0; } } -async function fetchSeedTicks({ apiBase, readToken, symbol, source, limit }) { +async function fetchSeedTicksPage({ apiBase, readToken, symbol, source, limit, to }) { const u = urlWithPath(apiBase, '/v1/ticks'); u.searchParams.set('symbol', symbol); u.searchParams.set('limit', String(limit)); if (source) u.searchParams.set('source', source); + if (to) u.searchParams.set('to', to); const res = await httpJson(u.toString(), { method: 'GET', @@ -182,6 +192,44 @@ async function fetchSeedTicks({ apiBase, readToken, symbol, source, limit }) { return ticks; } +async function fetchSeedTicksPaged({ apiBase, readToken, symbol, source, desiredLimit, pageLimit, maxPages }) { + const pages = []; + let cursorTo = null; + + for (let page = 0; page < maxPages; page++) { + const remaining = desiredLimit - pages.reduce((acc, p) => acc + p.length, 0); + if (remaining <= 0) break; + + const limit = Math.min(pageLimit, remaining); + const ticks = await fetchSeedTicksPage({ apiBase, readToken, symbol, source, limit, to: cursorTo }); + if (!ticks.length) break; + + // Server returns ascending ticks; we unshift to keep overall chronological order. + pages.unshift(ticks); + + const oldestTs = String(ticks[0]?.ts || '').trim(); + if (!oldestTs) break; + const oldestMs = Date.parse(oldestTs); + if (!Number.isFinite(oldestMs)) break; + cursorTo = new Date(oldestMs - 1).toISOString(); + } + + const flat = pages.flat(); + if (!flat.length) return flat; + + // Best-effort de-duplication (in case of overlapping `to` bounds). + const seen = new Set(); + const out = []; + for (const t of flat) { + const ts = String(t?.ts || ''); + const key = ts ? ts : JSON.stringify(t); + if (seen.has(key)) continue; + seen.add(key); + out.push(t); + } + return out; +} + async function ingestTick({ apiBase, writeToken, tick }) { const u = urlWithPath(apiBase, '/v1/ingest/tick'); const res = await httpJson(u.toString(), { @@ -218,23 +266,35 @@ async function main() { const readTokenFile = envString('FAKE_READ_TOKEN_FILE', envString('READ_TOKEN_FILE', '/tokens/read.json')); const writeTokenFile = envString('FAKE_WRITE_TOKEN_FILE', envString('WRITE_TOKEN_FILE', '/app/tokens/alg.json')); - const seedLimit = envInt('FAKE_SEED_LIMIT', 5000, { min: 50, max: 5000 }); + const seedLimitDesired = envInt('FAKE_SEED_LIMIT', 50_000, { min: 50, max: 200_000 }); + const seedPageLimit = envInt('FAKE_SEED_PAGE_LIMIT', 5000, { min: 50, max: 5000 }); + const seedMaxPages = envInt( + 'FAKE_SEED_MAX_PAGES', + Math.ceil(seedLimitDesired / seedPageLimit) + 2, + { min: 1, max: 200 } + ); const seedSourceRaw = process.env.FAKE_SEED_SOURCE; const seedSource = seedSourceRaw == null ? source : String(seedSourceRaw).trim(); - const minBlock = envInt('FAKE_BLOCK_MIN', 30, { min: 1, max: 5000 }); - const maxBlock = envInt('FAKE_BLOCK_MAX', 240, { min: 1, max: 5000 }); + const minBlock = envInt('FAKE_BLOCK_MIN', 120, { min: 1, max: 50_000 }); + const maxBlock = envInt('FAKE_BLOCK_MAX', 1200, { min: 1, max: 50_000 }); const volScale = envNumber('FAKE_VOL_SCALE', 1); - const noiseScale = envNumber('FAKE_NOISE_SCALE', 0.15); - const meanReversion = envNumber('FAKE_MEAN_REVERSION', 0.001); - const markNoiseBps = envNumber('FAKE_MARK_NOISE_BPS', 2); + const noiseScale = envNumber('FAKE_NOISE_SCALE', 0.05); + const meanReversion = envNumber('FAKE_MEAN_REVERSION', 0.0002); + const markNoiseBps = envNumber('FAKE_MARK_NOISE_BPS', 5); const logEvery = envInt('FAKE_LOG_EVERY', 30, { min: 1, max: 10_000 }); const marketIndexEnv = process.env.MARKET_INDEX; const marketIndexFallback = Number.isInteger(Number(marketIndexEnv)) ? Number(marketIndexEnv) : 0; const startPriceEnv = envNumber('FAKE_START_PRICE', 0); + const clampEnabled = isTruthy(process.env.FAKE_CLAMP ?? '1'); + const clampQLow = clamp(envNumber('FAKE_CLAMP_Q_LOW', 0.05), 0.0, 0.49); + const clampQHigh = clamp(envNumber('FAKE_CLAMP_Q_HIGH', 0.95), 0.51, 1.0); + const clampLowMult = envNumber('FAKE_CLAMP_LOW_MULT', 0.8); + const clampHighMult = envNumber('FAKE_CLAMP_HIGH_MULT', 1.2); + const readToken = readTokenFromFile(readTokenFile); const writeToken = readTokenFromFile(writeTokenFile); if (!writeToken) throw new Error(`Missing write token (expected JSON token at ${writeTokenFile})`); @@ -248,12 +308,14 @@ async function main() { if (!readToken) { console.warn(`[fake-ingestor] No read token at ${readTokenFile}; running without seed data.`); } else { - seedTicks = await fetchSeedTicks({ + seedTicks = await fetchSeedTicksPaged({ apiBase, readToken, symbol, source: seedSource ? seedSource : undefined, - limit: seedLimit, + desiredLimit: seedLimitDesired, + pageLimit: seedPageLimit, + maxPages: seedMaxPages, }); seedFromTs = seedTicks.length ? String(seedTicks[0]?.ts || '') : null; @@ -290,11 +352,11 @@ async function main() { const sampler = new BlockSampler(returns.length ? returns : [0], { minBlock, maxBlock }); - const p05 = quantile(seedPrices, 0.05); + const pLow = quantile(seedPrices, clampQLow); const p50 = quantile(seedPrices, 0.5); - const p95 = quantile(seedPrices, 0.95); - const clampMin = p05 > 0 ? p05 * 0.8 : Math.min(...seedPrices) * 0.8; - const clampMax = p95 > 0 ? p95 * 1.2 : Math.max(...seedPrices) * 1.2; + const pHigh = quantile(seedPrices, clampQHigh); + const clampMin = pLow > 0 ? pLow * clampLowMult : Math.min(...seedPrices) * clampLowMult; + const clampMax = pHigh > 0 ? pHigh * clampHighMult : Math.max(...seedPrices) * clampHighMult; let logPrice = Math.log(seedPrices[seedPrices.length - 1]); const targetLog = Math.log(p50 > 0 ? p50 : seedPrices[seedPrices.length - 1]); @@ -310,7 +372,9 @@ async function main() { marketIndex, seed: { ok: Boolean(seedTicks.length), - limit: seedLimit, + desiredLimit: seedLimitDesired, + pageLimit: seedPageLimit, + maxPages: seedMaxPages, source: seedSource || null, ticks: seedTicks.length, prices: seedPrices.length, @@ -326,7 +390,7 @@ async function main() { noiseScale, meanReversion, block: { minBlock, maxBlock }, - clamp: { min: clampMin, max: clampMax }, + clamp: clampEnabled ? { min: clampMin, max: clampMax } : { disabled: true }, }, }, null, @@ -357,7 +421,7 @@ async function main() { logPrice += step; let oraclePrice = Math.exp(logPrice); - if (Number.isFinite(clampMin) && Number.isFinite(clampMax) && clampMax > clampMin) { + if (clampEnabled && Number.isFinite(clampMin) && Number.isFinite(clampMax) && clampMax > clampMin) { oraclePrice = clamp(oraclePrice, clampMin, clampMax); logPrice = Math.log(oraclePrice); } @@ -377,8 +441,24 @@ async function main() { fake: true, model: 'block-bootstrap-returns', seeded: seedTicks.length - ? { symbol, source: seedSource || null, limit: seedLimit, fromTs: seedFromTs, toTs: seedToTs } - : { symbol, source: null, limit: 0, fromTs: null, toTs: null }, + ? { + symbol, + source: seedSource || null, + desiredLimit: seedLimitDesired, + pageLimit: seedPageLimit, + maxPages: seedMaxPages, + fromTs: seedFromTs, + toTs: seedToTs, + } + : { + symbol, + source: null, + desiredLimit: 0, + pageLimit: seedPageLimit, + maxPages: seedMaxPages, + fromTs: null, + toTs: null, + }, }, }; diff --git a/kustomize/overlays/staging/ingestor-fake-patch.yaml b/kustomize/overlays/staging/ingestor-fake-patch.yaml index ab8006e..f601924 100644 --- a/kustomize/overlays/staging/ingestor-fake-patch.yaml +++ b/kustomize/overlays/staging/ingestor-fake-patch.yaml @@ -13,7 +13,29 @@ spec: - name: FAKE_WRITE_TOKEN_FILE value: "/app/tokens/alg.json" - name: FAKE_SEED_LIMIT + value: "50000" + - name: FAKE_SEED_PAGE_LIMIT value: "5000" + - name: FAKE_SEED_MAX_PAGES + value: "20" + - name: FAKE_BLOCK_MIN + value: "300" + - name: FAKE_BLOCK_MAX + value: "1800" + - name: FAKE_VOL_SCALE + value: "1.8" + - name: FAKE_NOISE_SCALE + value: "0.03" + - name: FAKE_MEAN_REVERSION + value: "0.0001" + - name: FAKE_CLAMP_Q_LOW + value: "0.02" + - name: FAKE_CLAMP_Q_HIGH + value: "0.98" + - name: FAKE_CLAMP_LOW_MULT + value: "0.7" + - name: FAKE_CLAMP_HIGH_MULT + value: "1.3" command: ["node"] args: ["/opt/fake/fake-ingestor.mjs"] volumeMounts: