feat(staging): tune fake-ingestor realism

- Seed uses paged /v1/ticks fetch (beyond 5k) to capture longer history.

- Longer return blocks + adjusted volatility params for a more lifelike curve.

kustomize/overlays/staging/fake-ingestor.mjs

@@ -99,6 +99,14 @@ function clamp(v, min, max) {
   return v;
 }
 
+function isTruthy(value) {
+  const v = String(value ?? '')
+    .trim()
+    .toLowerCase();
+  if (!v) return false;
+  return !['0', 'false', 'off', 'no', 'none', 'disabled'].includes(v);
+}
+
 function parsePriceFromTick(t) {
   const mp = t?.mark_price ?? t?.markPrice;
   const op = t?.oracle_price ?? t?.oraclePrice ?? t?.price;
@@ -123,6 +131,7 @@ class BlockSampler {
   #minBlock;
   #maxBlock;
   #idx = 0;
+  #end = 0;
   #remaining = 0;
 
   constructor(values, { minBlock, maxBlock }) {
@@ -136,7 +145,8 @@ class BlockSampler {
     const n = this.#values.length;
     if (n <= 1) {
       this.#idx = 0;
-      this.#remaining = 1;
+      this.#end = n;
+      this.#remaining = n;
       return;
     }
 
@@ -145,28 +155,28 @@ class BlockSampler {
     const len = this.#minBlock + Math.floor(Math.random() * span);
 
     this.#idx = start;
-    this.#remaining = Math.min(len, n);
+    this.#end = Math.min(n, start + len);
+    this.#remaining = this.#end - this.#idx;
   }
 
   next() {
-    const n = this.#values.length;
+    if (this.#values.length === 0) return 0;
-    if (n === 0) return 0;
+    if (this.#remaining <= 0 || this.#idx >= this.#end) this.#startNewBlock();
 
-    if (this.#remaining <= 0) this.#startNewBlock();
     const v = this.#values[this.#idx];
-
+    this.#idx += 1;
-    this.#idx = (this.#idx + 1) % n;
     this.#remaining -= 1;
 
     return Number.isFinite(v) ? v : 0;
   }
 }
 
-async function fetchSeedTicks({ apiBase, readToken, symbol, source, limit }) {
+async function fetchSeedTicksPage({ apiBase, readToken, symbol, source, limit, to }) {
   const u = urlWithPath(apiBase, '/v1/ticks');
   u.searchParams.set('symbol', symbol);
   u.searchParams.set('limit', String(limit));
   if (source) u.searchParams.set('source', source);
+  if (to) u.searchParams.set('to', to);
 
   const res = await httpJson(u.toString(), {
     method: 'GET',
@@ -182,6 +192,44 @@ async function fetchSeedTicks({ apiBase, readToken, symbol, source, limit }) {
   return ticks;
 }
 
+async function fetchSeedTicksPaged({ apiBase, readToken, symbol, source, desiredLimit, pageLimit, maxPages }) {
+  const pages = [];
+  let cursorTo = null;
+
+  for (let page = 0; page < maxPages; page++) {
+    const remaining = desiredLimit - pages.reduce((acc, p) => acc + p.length, 0);
+    if (remaining <= 0) break;
+
+    const limit = Math.min(pageLimit, remaining);
+    const ticks = await fetchSeedTicksPage({ apiBase, readToken, symbol, source, limit, to: cursorTo });
+    if (!ticks.length) break;
+
+    // Server returns ascending ticks; we unshift to keep overall chronological order.
+    pages.unshift(ticks);
+
+    const oldestTs = String(ticks[0]?.ts || '').trim();
+    if (!oldestTs) break;
+    const oldestMs = Date.parse(oldestTs);
+    if (!Number.isFinite(oldestMs)) break;
+    cursorTo = new Date(oldestMs - 1).toISOString();
+  }
+
+  const flat = pages.flat();
+  if (!flat.length) return flat;
+
+  // Best-effort de-duplication (in case of overlapping `to` bounds).
+  const seen = new Set();
+  const out = [];
+  for (const t of flat) {
+    const ts = String(t?.ts || '');
+    const key = ts ? ts : JSON.stringify(t);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(t);
+  }
+  return out;
+}
+
 async function ingestTick({ apiBase, writeToken, tick }) {
   const u = urlWithPath(apiBase, '/v1/ingest/tick');
   const res = await httpJson(u.toString(), {
@@ -218,23 +266,35 @@ async function main() {
   const readTokenFile = envString('FAKE_READ_TOKEN_FILE', envString('READ_TOKEN_FILE', '/tokens/read.json'));
   const writeTokenFile = envString('FAKE_WRITE_TOKEN_FILE', envString('WRITE_TOKEN_FILE', '/app/tokens/alg.json'));
 
-  const seedLimit = envInt('FAKE_SEED_LIMIT', 5000, { min: 50, max: 5000 });
+  const seedLimitDesired = envInt('FAKE_SEED_LIMIT', 50_000, { min: 50, max: 200_000 });
+  const seedPageLimit = envInt('FAKE_SEED_PAGE_LIMIT', 5000, { min: 50, max: 5000 });
+  const seedMaxPages = envInt(
+    'FAKE_SEED_MAX_PAGES',
+    Math.ceil(seedLimitDesired / seedPageLimit) + 2,
+    { min: 1, max: 200 }
+  );
   const seedSourceRaw = process.env.FAKE_SEED_SOURCE;
   const seedSource = seedSourceRaw == null ? source : String(seedSourceRaw).trim();
 
-  const minBlock = envInt('FAKE_BLOCK_MIN', 30, { min: 1, max: 5000 });
+  const minBlock = envInt('FAKE_BLOCK_MIN', 120, { min: 1, max: 50_000 });
-  const maxBlock = envInt('FAKE_BLOCK_MAX', 240, { min: 1, max: 5000 });
+  const maxBlock = envInt('FAKE_BLOCK_MAX', 1200, { min: 1, max: 50_000 });
 
   const volScale = envNumber('FAKE_VOL_SCALE', 1);
-  const noiseScale = envNumber('FAKE_NOISE_SCALE', 0.15);
+  const noiseScale = envNumber('FAKE_NOISE_SCALE', 0.05);
-  const meanReversion = envNumber('FAKE_MEAN_REVERSION', 0.001);
+  const meanReversion = envNumber('FAKE_MEAN_REVERSION', 0.0002);
-  const markNoiseBps = envNumber('FAKE_MARK_NOISE_BPS', 2);
+  const markNoiseBps = envNumber('FAKE_MARK_NOISE_BPS', 5);
   const logEvery = envInt('FAKE_LOG_EVERY', 30, { min: 1, max: 10_000 });
 
   const marketIndexEnv = process.env.MARKET_INDEX;
   const marketIndexFallback = Number.isInteger(Number(marketIndexEnv)) ? Number(marketIndexEnv) : 0;
   const startPriceEnv = envNumber('FAKE_START_PRICE', 0);
 
+  const clampEnabled = isTruthy(process.env.FAKE_CLAMP ?? '1');
+  const clampQLow = clamp(envNumber('FAKE_CLAMP_Q_LOW', 0.05), 0.0, 0.49);
+  const clampQHigh = clamp(envNumber('FAKE_CLAMP_Q_HIGH', 0.95), 0.51, 1.0);
+  const clampLowMult = envNumber('FAKE_CLAMP_LOW_MULT', 0.8);
+  const clampHighMult = envNumber('FAKE_CLAMP_HIGH_MULT', 1.2);
+
   const readToken = readTokenFromFile(readTokenFile);
   const writeToken = readTokenFromFile(writeTokenFile);
   if (!writeToken) throw new Error(`Missing write token (expected JSON token at ${writeTokenFile})`);
@@ -248,12 +308,14 @@ async function main() {
   if (!readToken) {
     console.warn(`[fake-ingestor] No read token at ${readTokenFile}; running without seed data.`);
   } else {
-    seedTicks = await fetchSeedTicks({
+    seedTicks = await fetchSeedTicksPaged({
       apiBase,
       readToken,
       symbol,
       source: seedSource ? seedSource : undefined,
-      limit: seedLimit,
+      desiredLimit: seedLimitDesired,
+      pageLimit: seedPageLimit,
+      maxPages: seedMaxPages,
     });
 
     seedFromTs = seedTicks.length ? String(seedTicks[0]?.ts || '') : null;
@@ -290,11 +352,11 @@ async function main() {
 
   const sampler = new BlockSampler(returns.length ? returns : [0], { minBlock, maxBlock });
 
-  const p05 = quantile(seedPrices, 0.05);
+  const pLow = quantile(seedPrices, clampQLow);
   const p50 = quantile(seedPrices, 0.5);
-  const p95 = quantile(seedPrices, 0.95);
+  const pHigh = quantile(seedPrices, clampQHigh);
-  const clampMin = p05 > 0 ? p05 * 0.8 : Math.min(...seedPrices) * 0.8;
+  const clampMin = pLow > 0 ? pLow * clampLowMult : Math.min(...seedPrices) * clampLowMult;
-  const clampMax = p95 > 0 ? p95 * 1.2 : Math.max(...seedPrices) * 1.2;
+  const clampMax = pHigh > 0 ? pHigh * clampHighMult : Math.max(...seedPrices) * clampHighMult;
 
   let logPrice = Math.log(seedPrices[seedPrices.length - 1]);
   const targetLog = Math.log(p50 > 0 ? p50 : seedPrices[seedPrices.length - 1]);
@@ -310,7 +372,9 @@ async function main() {
         marketIndex,
         seed: {
           ok: Boolean(seedTicks.length),
-          limit: seedLimit,
+          desiredLimit: seedLimitDesired,
+          pageLimit: seedPageLimit,
+          maxPages: seedMaxPages,
           source: seedSource || null,
           ticks: seedTicks.length,
           prices: seedPrices.length,
@@ -326,7 +390,7 @@ async function main() {
           noiseScale,
           meanReversion,
           block: { minBlock, maxBlock },
-          clamp: { min: clampMin, max: clampMax },
+          clamp: clampEnabled ? { min: clampMin, max: clampMax } : { disabled: true },
         },
       },
       null,
@@ -357,7 +421,7 @@ async function main() {
 
     logPrice += step;
     let oraclePrice = Math.exp(logPrice);
-    if (Number.isFinite(clampMin) && Number.isFinite(clampMax) && clampMax > clampMin) {
+    if (clampEnabled && Number.isFinite(clampMin) && Number.isFinite(clampMax) && clampMax > clampMin) {
       oraclePrice = clamp(oraclePrice, clampMin, clampMax);
       logPrice = Math.log(oraclePrice);
     }
@@ -377,8 +441,24 @@ async function main() {
         fake: true,
         model: 'block-bootstrap-returns',
         seeded: seedTicks.length
-          ? { symbol, source: seedSource || null, limit: seedLimit, fromTs: seedFromTs, toTs: seedToTs }
+          ? {
-          : { symbol, source: null, limit: 0, fromTs: null, toTs: null },
+              symbol,
+              source: seedSource || null,
+              desiredLimit: seedLimitDesired,
+              pageLimit: seedPageLimit,
+              maxPages: seedMaxPages,
+              fromTs: seedFromTs,
+              toTs: seedToTs,
+            }
+          : {
+              symbol,
+              source: null,
+              desiredLimit: 0,
+              pageLimit: seedPageLimit,
+              maxPages: seedMaxPages,
+              fromTs: null,
+              toTs: null,
+            },
       },
     };
 

kustomize/overlays/staging/ingestor-fake-patch.yaml

@@ -13,7 +13,29 @@ spec:
             - name: FAKE_WRITE_TOKEN_FILE
               value: "/app/tokens/alg.json"
             - name: FAKE_SEED_LIMIT
+              value: "50000"
+            - name: FAKE_SEED_PAGE_LIMIT
               value: "5000"
+            - name: FAKE_SEED_MAX_PAGES
+              value: "20"
+            - name: FAKE_BLOCK_MIN
+              value: "300"
+            - name: FAKE_BLOCK_MAX
+              value: "1800"
+            - name: FAKE_VOL_SCALE
+              value: "1.8"
+            - name: FAKE_NOISE_SCALE
+              value: "0.03"
+            - name: FAKE_MEAN_REVERSION
+              value: "0.0001"
+            - name: FAKE_CLAMP_Q_LOW
+              value: "0.02"
+            - name: FAKE_CLAMP_Q_HIGH
+              value: "0.98"
+            - name: FAKE_CLAMP_LOW_MULT
+              value: "0.7"
+            - name: FAKE_CLAMP_HIGH_MULT
+              value: "1.3"
           command: ["node"]
           args: ["/opt/fake/fake-ingestor.mjs"]
           volumeMounts: