trade/trade-deploy
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(staging): tune fake-ingestor realism

Browse Source 
- Seed uses paged /v1/ticks fetch (beyond 5k) to capture longer history.

- Longer return blocks + adjusted volatility params for a more lifelike curve.
This commit is contained in:
u1
2026-01-09 01:48:55 +01:00
parent 0eef6bca12
commit 75e87a7cc8
2 changed files with 127 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
kustomize/overlays/staging/fake-ingestor.mjs
View File

@@ -99,6 +99,14 @@ function clamp(v, min, max) {
return v;
}
function isTruthy(value) {
const v = String(value ?? '')
.trim()
.toLowerCase();
if (!v) return false;
return !['0', 'false', 'off', 'no', 'none', 'disabled'].includes(v);
}
function parsePriceFromTick(t) {
const mp = t?.mark_price ?? t?.markPrice;
const op = t?.oracle_price ?? t?.oraclePrice ?? t?.price;
@@ -123,6 +131,7 @@ class BlockSampler {
#minBlock;
#maxBlock;
#idx = 0;
#end = 0;
#remaining = 0;
constructor(values, { minBlock, maxBlock }) {
@@ -136,7 +145,8 @@ class BlockSampler {
const n = this.#values.length;
if (n <= 1) {
this.#idx = 0;
this.#remaining = 1;
this.#end = n;
this.#remaining = n;
return;
}
@@ -145,28 +155,28 @@ class BlockSampler {
const len = this.#minBlock + Math.floor(Math.random() * span);
this.#idx = start;
this.#remaining = Math.min(len, n);
this.#end = Math.min(n, start + len);
this.#remaining = this.#end - this.#idx;
}
next() {
const n = this.#values.length;
if (n === 0) return 0;
if (this.#values.length === 0) return 0;
if (this.#remaining <= 0 || this.#idx >= this.#end) this.#startNewBlock();
if (this.#remaining <= 0) this.#startNewBlock();
const v = this.#values[this.#idx];
this.#idx = (this.#idx + 1) % n;
this.#idx += 1;
this.#remaining -= 1;
return Number.isFinite(v) ? v : 0;
}
}
async function fetchSeedTicks({ apiBase, readToken, symbol, source, limit }) {
async function fetchSeedTicksPage({ apiBase, readToken, symbol, source, limit, to }) {
const u = urlWithPath(apiBase, '/v1/ticks');
u.searchParams.set('symbol', symbol);
u.searchParams.set('limit', String(limit));
if (source) u.searchParams.set('source', source);
if (to) u.searchParams.set('to', to);
const res = await httpJson(u.toString(), {
method: 'GET',
@@ -182,6 +192,44 @@ async function fetchSeedTicks({ apiBase, readToken, symbol, source, limit }) {
return ticks;
}
async function fetchSeedTicksPaged({ apiBase, readToken, symbol, source, desiredLimit, pageLimit, maxPages }) {
const pages = [];
let cursorTo = null;
for (let page = 0; page < maxPages; page++) {
const remaining = desiredLimit - pages.reduce((acc, p) => acc + p.length, 0);
if (remaining <= 0) break;
const limit = Math.min(pageLimit, remaining);
const ticks = await fetchSeedTicksPage({ apiBase, readToken, symbol, source, limit, to: cursorTo });
if (!ticks.length) break;
// Server returns ascending ticks; we unshift to keep overall chronological order.
pages.unshift(ticks);
const oldestTs = String(ticks[0]?.ts || '').trim();
if (!oldestTs) break;
const oldestMs = Date.parse(oldestTs);
if (!Number.isFinite(oldestMs)) break;
cursorTo = new Date(oldestMs - 1).toISOString();
}
const flat = pages.flat();
if (!flat.length) return flat;
// Best-effort de-duplication (in case of overlapping `to` bounds).
const seen = new Set();
const out = [];
for (const t of flat) {
const ts = String(t?.ts || '');
const key = ts ? ts : JSON.stringify(t);
if (seen.has(key)) continue;
seen.add(key);
out.push(t);
}
return out;
}
async function ingestTick({ apiBase, writeToken, tick }) {
const u = urlWithPath(apiBase, '/v1/ingest/tick');
const res = await httpJson(u.toString(), {
@@ -218,23 +266,35 @@ async function main() {
const readTokenFile = envString('FAKE_READ_TOKEN_FILE', envString('READ_TOKEN_FILE', '/tokens/read.json'));
const writeTokenFile = envString('FAKE_WRITE_TOKEN_FILE', envString('WRITE_TOKEN_FILE', '/app/tokens/alg.json'));
const seedLimit = envInt('FAKE_SEED_LIMIT', 5000, { min: 50, max: 5000 });
const seedLimitDesired = envInt('FAKE_SEED_LIMIT', 50_000, { min: 50, max: 200_000 });
const seedPageLimit = envInt('FAKE_SEED_PAGE_LIMIT', 5000, { min: 50, max: 5000 });
const seedMaxPages = envInt(
'FAKE_SEED_MAX_PAGES',
Math.ceil(seedLimitDesired / seedPageLimit) + 2,
{ min: 1, max: 200 }
);
const seedSourceRaw = process.env.FAKE_SEED_SOURCE;
const seedSource = seedSourceRaw == null ? source : String(seedSourceRaw).trim();
const minBlock = envInt('FAKE_BLOCK_MIN', 30, { min: 1, max: 5000 });
const maxBlock = envInt('FAKE_BLOCK_MAX', 240, { min: 1, max: 5000 });
const minBlock = envInt('FAKE_BLOCK_MIN', 120, { min: 1, max: 50_000 });
const maxBlock = envInt('FAKE_BLOCK_MAX', 1200, { min: 1, max: 50_000 });
const volScale = envNumber('FAKE_VOL_SCALE', 1);
const noiseScale = envNumber('FAKE_NOISE_SCALE', 0.15);
const meanReversion = envNumber('FAKE_MEAN_REVERSION', 0.001);
const markNoiseBps = envNumber('FAKE_MARK_NOISE_BPS', 2);
const noiseScale = envNumber('FAKE_NOISE_SCALE', 0.05);
const meanReversion = envNumber('FAKE_MEAN_REVERSION', 0.0002);
const markNoiseBps = envNumber('FAKE_MARK_NOISE_BPS', 5);
const logEvery = envInt('FAKE_LOG_EVERY', 30, { min: 1, max: 10_000 });
const marketIndexEnv = process.env.MARKET_INDEX;
const marketIndexFallback = Number.isInteger(Number(marketIndexEnv)) ? Number(marketIndexEnv) : 0;
const startPriceEnv = envNumber('FAKE_START_PRICE', 0);
const clampEnabled = isTruthy(process.env.FAKE_CLAMP ?? '1');
const clampQLow = clamp(envNumber('FAKE_CLAMP_Q_LOW', 0.05), 0.0, 0.49);
const clampQHigh = clamp(envNumber('FAKE_CLAMP_Q_HIGH', 0.95), 0.51, 1.0);
const clampLowMult = envNumber('FAKE_CLAMP_LOW_MULT', 0.8);
const clampHighMult = envNumber('FAKE_CLAMP_HIGH_MULT', 1.2);
const readToken = readTokenFromFile(readTokenFile);
const writeToken = readTokenFromFile(writeTokenFile);
if (!writeToken) throw new Error(`Missing write token (expected JSON token at ${writeTokenFile})`);
@@ -248,12 +308,14 @@ async function main() {
if (!readToken) {
console.warn(`[fake-ingestor] No read token at ${readTokenFile}; running without seed data.`);
} else {
seedTicks = await fetchSeedTicks({
seedTicks = await fetchSeedTicksPaged({
apiBase,
readToken,
symbol,
source: seedSource ? seedSource : undefined,
limit: seedLimit,
desiredLimit: seedLimitDesired,
pageLimit: seedPageLimit,
maxPages: seedMaxPages,
});
seedFromTs = seedTicks.length ? String(seedTicks[0]?.ts || '') : null;
@@ -290,11 +352,11 @@ async function main() {
const sampler = new BlockSampler(returns.length ? returns : [0], { minBlock, maxBlock });
const p05 = quantile(seedPrices, 0.05);
const pLow = quantile(seedPrices, clampQLow);
const p50 = quantile(seedPrices, 0.5);
const p95 = quantile(seedPrices, 0.95);
const clampMin = p05 > 0 ? p05 * 0.8 : Math.min(...seedPrices) * 0.8;
const clampMax = p95 > 0 ? p95 * 1.2 : Math.max(...seedPrices) * 1.2;
const pHigh = quantile(seedPrices, clampQHigh);
const clampMin = pLow > 0 ? pLow * clampLowMult : Math.min(...seedPrices) * clampLowMult;
const clampMax = pHigh > 0 ? pHigh * clampHighMult : Math.max(...seedPrices) * clampHighMult;
let logPrice = Math.log(seedPrices[seedPrices.length - 1]);
const targetLog = Math.log(p50 > 0 ? p50 : seedPrices[seedPrices.length - 1]);
@@ -310,7 +372,9 @@ async function main() {
marketIndex,
seed: {
ok: Boolean(seedTicks.length),
limit: seedLimit,
desiredLimit: seedLimitDesired,
pageLimit: seedPageLimit,
maxPages: seedMaxPages,
source: seedSource || null,
ticks: seedTicks.length,
prices: seedPrices.length,
@@ -326,7 +390,7 @@ async function main() {
noiseScale,
meanReversion,
block: { minBlock, maxBlock },
clamp: { min: clampMin, max: clampMax },
clamp: clampEnabled ? { min: clampMin, max: clampMax } : { disabled: true },
},
},
null,
@@ -357,7 +421,7 @@ async function main() {
logPrice += step;
let oraclePrice = Math.exp(logPrice);
if (Number.isFinite(clampMin) && Number.isFinite(clampMax) && clampMax > clampMin) {
if (clampEnabled && Number.isFinite(clampMin) && Number.isFinite(clampMax) && clampMax > clampMin) {
oraclePrice = clamp(oraclePrice, clampMin, clampMax);
logPrice = Math.log(oraclePrice);
}
@@ -377,8 +441,24 @@ async function main() {
fake: true,
model: 'block-bootstrap-returns',
seeded: seedTicks.length
? { symbol, source: seedSource || null, limit: seedLimit, fromTs: seedFromTs, toTs: seedToTs }
: { symbol, source: null, limit: 0, fromTs: null, toTs: null },
? {
symbol,
source: seedSource || null,
desiredLimit: seedLimitDesired,
pageLimit: seedPageLimit,
maxPages: seedMaxPages,
fromTs: seedFromTs,
toTs: seedToTs,
}
: {
symbol,
source: null,
desiredLimit: 0,
pageLimit: seedPageLimit,
maxPages: seedMaxPages,
fromTs: null,
toTs: null,
},
},
};

22
kustomize/overlays/staging/ingestor-fake-patch.yaml
View File

@@ -13,7 +13,29 @@ spec:
- name: FAKE_WRITE_TOKEN_FILE
value: "/app/tokens/alg.json"
- name: FAKE_SEED_LIMIT
value: "50000"
- name: FAKE_SEED_PAGE_LIMIT
value: "5000"
- name: FAKE_SEED_MAX_PAGES
value: "20"
- name: FAKE_BLOCK_MIN
value: "300"
- name: FAKE_BLOCK_MAX
value: "1800"
- name: FAKE_VOL_SCALE
value: "1.8"
- name: FAKE_NOISE_SCALE
value: "0.03"
- name: FAKE_MEAN_REVERSION
value: "0.0001"
- name: FAKE_CLAMP_Q_LOW
value: "0.02"
- name: FAKE_CLAMP_Q_HIGH
value: "0.98"
- name: FAKE_CLAMP_LOW_MULT
value: "0.7"
- name: FAKE_CLAMP_HIGH_MULT
value: "1.3"
command: ["node"]
args: ["/opt/fake/fake-ingestor.mjs"]
volumeMounts: