From 0661e55c878a8268e87e0f24aa8297e9c841ecb8 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 22 Feb 2024 22:07:40 +0000 Subject: [PATCH 1/4] Bumping sdk dependency to 2.65.0-beta.7 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 6e10052..300100b 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "license": "Apache-2.0", "dependencies": { "@coral-xyz/anchor": "^0.29.0", - "@drift-labs/sdk": "2.65.0-beta.6", + "@drift-labs/sdk": "2.65.0-beta.7", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index b404613..251b50e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -115,10 +115,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.65.0-beta.6": - version "2.65.0-beta.6" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.65.0-beta.6.tgz#7da1434ce04142f4adca573baf44dc3539303a0f" - integrity sha512-ENnkk1o1wahXoT3F1svjiHzVdSrxlVZ7NfUXmrtyQEqbcZtbUyoS6CqL9f7aC273ydw7xC0a3jrxEH0yMOrcVQ== +"@drift-labs/sdk@2.65.0-beta.7": + version "2.65.0-beta.7" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.65.0-beta.7.tgz#90c07cf2ede6407cb5c098da06b94b97640a4c12" + integrity sha512-wZqz7GGhJ9zoOYCdgSjZjDdW39FTOVHqHsspRp8dPksp8NYlM2aaqBFUwDl4wZtDIjLzquY4K37NSf6O93JN0A== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 679757c4eccca1e305fa9568122038cca8a68d5f Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 23 Feb 2024 20:48:41 +0000 Subject: [PATCH 2/4] Bumping sdk dependency to 2.65.0-beta.8 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 300100b..3c0ed1a 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "license": "Apache-2.0", "dependencies": { "@coral-xyz/anchor": "^0.29.0", - "@drift-labs/sdk": "2.65.0-beta.7", + "@drift-labs/sdk": "2.65.0-beta.8", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 251b50e..662ca4a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -115,10 +115,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.65.0-beta.7": - version "2.65.0-beta.7" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.65.0-beta.7.tgz#90c07cf2ede6407cb5c098da06b94b97640a4c12" - integrity sha512-wZqz7GGhJ9zoOYCdgSjZjDdW39FTOVHqHsspRp8dPksp8NYlM2aaqBFUwDl4wZtDIjLzquY4K37NSf6O93JN0A== +"@drift-labs/sdk@2.65.0-beta.8": + version "2.65.0-beta.8" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.65.0-beta.8.tgz#c7b501a9bb54c0ab82e96da4cb5b865b9e4ef7ef" + integrity sha512-tIfC4UatP1oOkyLm9s66J/bFiUhk9hyc1418ZCQPF5QA31do3r8y9yyajzbvUjdotppoPTjPt8djYtyqSOAr7w== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 5401cfefd53d625cbfbf10808c3ed0ca386ca7fd Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 24 Feb 2024 05:41:05 +0000 Subject: [PATCH 3/4] Bumping sdk dependency to 2.65.0-beta.9 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 3c0ed1a..91931da 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "license": "Apache-2.0", "dependencies": { "@coral-xyz/anchor": "^0.29.0", - "@drift-labs/sdk": "2.65.0-beta.8", + "@drift-labs/sdk": "2.65.0-beta.9", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 662ca4a..c55a508 100644 --- a/yarn.lock +++ b/yarn.lock @@ -115,10 +115,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.65.0-beta.8": - version "2.65.0-beta.8" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.65.0-beta.8.tgz#c7b501a9bb54c0ab82e96da4cb5b865b9e4ef7ef" - integrity sha512-tIfC4UatP1oOkyLm9s66J/bFiUhk9hyc1418ZCQPF5QA31do3r8y9yyajzbvUjdotppoPTjPt8djYtyqSOAr7w== +"@drift-labs/sdk@2.65.0-beta.9": + version "2.65.0-beta.9" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.65.0-beta.9.tgz#d278dd7ca50fe65191faa01e7700d6e1daa29928" + integrity sha512-KmWtpW82cyl/tODPNBT+AbOQi8MIdxtLkLwuDyN6pHeVz8Op+Jie5C7iZXTzqaTwrJah8C0fFdsLfK0RWHWMtw== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From d035440e93444784dcf5834814831bf938db086a Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Sun, 25 Feb 2024 13:59:28 -0800 Subject: [PATCH 4/4] cache l3 and add metric --- src/core/metrics.ts | 9 ++ src/dlob-subscriber/DLOBSubscriberIO.ts | 176 +++++++++++++++++++----- src/index.ts | 69 +++++++--- 3 files changed, 199 insertions(+), 55 deletions(-) diff --git a/src/core/metrics.ts b/src/core/metrics.ts index 09e920f..574c245 100644 --- a/src/core/metrics.ts +++ b/src/core/metrics.ts @@ -37,6 +37,7 @@ enum METRIC_TYPES { cache_miss_count = 'cache_miss_count', current_system_ts = 'current_system_ts', health_status = 'health_status', + incoming_requests_count = 'incoming_requests_count', } export enum HEALTH_STATUS { @@ -116,6 +117,13 @@ const cacheHitCounter = meter.createCounter(METRIC_TYPES.cache_hit_count, { description: 'Total redis cache hits', }); +const incomingRequestsCounter = meter.createCounter( + METRIC_TYPES.incoming_requests_count, + { + description: 'Total incoming requests', + } +); + const accountUpdatesCounter = meter.createCounter( METRIC_TYPES.account_updates_count, { @@ -225,6 +233,7 @@ export { endpointResponseTimeHistogram, gpaFetchDurationHistogram, responseStatusCounter, + incomingRequestsCounter, handleHealthCheck, setLastReceivedWsMsgTs, accountUpdatesCounter, diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 2ef863f..64d2a54 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -15,7 +15,7 @@ import { l2WithBNToStrings, } from '../utils/utils'; -type wsMarketL2Args = { +type wsMarketArgs = { marketIndex: number; marketType: MarketType; marketName: string; @@ -36,7 +36,7 @@ export type wsMarketInfo = { }; export class DLOBSubscriberIO extends DLOBSubscriber { - public marketL2Args: wsMarketL2Args[] = []; + public marketArgs: wsMarketArgs[] = []; public lastSeenL2Formatted: Map>; redisClient: RedisClient; public killSwitchSlotDiffThreshold: number; @@ -73,7 +73,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { ); const includeVamm = !isVariant(perpMarket.status, 'ammPaused'); - this.marketL2Args.push({ + this.marketArgs.push({ marketIndex: market.marketIndex, marketType: MarketType.PERP, marketName: market.marketName, @@ -85,7 +85,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { }); } for (const market of config.spotMarketInfos) { - this.marketL2Args.push({ + this.marketArgs.push({ marketIndex: market.marketIndex, marketType: MarketType.SPOT, marketName: market.marketName, @@ -102,70 +102,75 @@ export class DLOBSubscriberIO extends DLOBSubscriber { override async updateDLOB(): Promise { await super.updateDLOB(); - for (const l2Args of this.marketL2Args) { + for (const marketArgs of this.marketArgs) { try { - this.getL2AndSendMsg(l2Args); + this.getL2AndSendMsg(marketArgs); + this.getL3AndSendMsg(marketArgs); } catch (error) { console.error(error); - console.log(`Error getting L2 ${l2Args.marketName}`); + console.log(`Error getting L2 ${marketArgs.marketName}`); } } } - getL2AndSendMsg(l2Args: wsMarketL2Args): void { - const grouping = l2Args.grouping; - const { marketName, ...l2FuncArgs } = l2Args; + getL2AndSendMsg(marketArgs: wsMarketArgs): void { + const grouping = marketArgs.grouping; + const { marketName, ...l2FuncArgs } = marketArgs; const l2 = this.getL2(l2FuncArgs); const slot = l2.slot; const lastMarketSlotAndTime = this.lastMarketSlotMap - .get(l2Args.marketType) - .get(l2Args.marketIndex); + .get(marketArgs.marketType) + .get(marketArgs.marketIndex); if (!lastMarketSlotAndTime) { this.lastMarketSlotMap - .get(l2Args.marketType) - .set(l2Args.marketIndex, { slot, ts: Date.now() }); + .get(marketArgs.marketType) + .set(marketArgs.marketIndex, { slot, ts: Date.now() }); } if (slot) { delete l2.slot; } - const marketType = isVariant(l2Args.marketType, 'perp') ? 'perp' : 'spot'; + const marketType = isVariant(marketArgs.marketType, 'perp') + ? 'perp' + : 'spot'; let l2Formatted: any; if (grouping) { const groupingBN = new BN(grouping); - l2Formatted = l2WithBNToStrings(groupL2(l2, groupingBN, l2Args.depth)); + l2Formatted = l2WithBNToStrings( + groupL2(l2, groupingBN, marketArgs.depth) + ); } else { l2Formatted = l2WithBNToStrings(l2); } - if (l2Args.updateOnChange) { + if (marketArgs.updateOnChange) { if ( this.lastSeenL2Formatted - .get(l2Args.marketType) - ?.get(l2Args.marketIndex) === JSON.stringify(l2Formatted) + .get(marketArgs.marketType) + ?.get(marketArgs.marketIndex) === JSON.stringify(l2Formatted) ) return; } this.lastSeenL2Formatted - .get(l2Args.marketType) - ?.set(l2Args.marketIndex, JSON.stringify(l2Formatted)); + .get(marketArgs.marketType) + ?.set(marketArgs.marketIndex, JSON.stringify(l2Formatted)); l2Formatted['marketName'] = marketName?.toUpperCase(); l2Formatted['marketType'] = marketType?.toLowerCase(); - l2Formatted['marketIndex'] = l2Args.marketIndex; + l2Formatted['marketIndex'] = marketArgs.marketIndex; l2Formatted['ts'] = Date.now(); l2Formatted['slot'] = slot; addOracletoResponse( l2Formatted, this.driftClient, - l2Args.marketType, - l2Args.marketIndex + marketArgs.marketType, + marketArgs.marketIndex ); addMarketSlotToResponse( l2Formatted, this.driftClient, - l2Args.marketType, - l2Args.marketIndex + marketArgs.marketType, + marketArgs.marketIndex ); // Check if slot diffs are too large for oracle @@ -200,12 +205,14 @@ export class DLOBSubscriberIO extends DLOBSubscriber { l2Formatted['marketSlot'] !== lastMarketSlotAndTime.slot ) { console.log( - `Updating market slot for ${l2Args.marketName} with slot ${l2Formatted['marketSlot']}` + `Updating market slot for ${marketArgs.marketName} with slot ${l2Formatted['marketSlot']}` ); - this.lastMarketSlotMap.get(l2Args.marketType).set(l2Args.marketIndex, { - slot: l2Formatted['marketSlot'], - ts: Date.now(), - }); + this.lastMarketSlotMap + .get(marketArgs.marketType) + .set(marketArgs.marketIndex, { + slot: l2Formatted['marketSlot'], + ts: Date.now(), + }); } const l2Formatted_depth100 = Object.assign({}, l2Formatted, { @@ -222,24 +229,121 @@ export class DLOBSubscriberIO extends DLOBSubscriber { }); this.redisClient.client.publish( - `orderbook_${marketType}_${l2Args.marketIndex}`, + `orderbook_${marketType}_${marketArgs.marketIndex}`, JSON.stringify(l2Formatted) ); this.redisClient.client.set( - `last_update_orderbook_${marketType}_${l2Args.marketIndex}`, + `last_update_orderbook_${marketType}_${marketArgs.marketIndex}`, JSON.stringify(l2Formatted_depth100) ); this.redisClient.client.set( - `last_update_orderbook_${marketType}_${l2Args.marketIndex}_depth_100`, + `last_update_orderbook_${marketType}_${marketArgs.marketIndex}_depth_100`, JSON.stringify(l2Formatted_depth100) ); this.redisClient.client.set( - `last_update_orderbook_${marketType}_${l2Args.marketIndex}_depth_20`, + `last_update_orderbook_${marketType}_${marketArgs.marketIndex}_depth_20`, JSON.stringify(l2Formatted_depth20) ); this.redisClient.client.set( - `last_update_orderbook_${marketType}_${l2Args.marketIndex}_depth_5`, + `last_update_orderbook_${marketType}_${marketArgs.marketIndex}_depth_5`, JSON.stringify(l2Formatted_depth5) ); } + + getL3AndSendMsg(marketArgs: wsMarketArgs): void { + const { marketName, ...l2FuncArgs } = marketArgs; + const l3 = this.getL3(l2FuncArgs); + const slot = l3.slot; + const lastMarketSlotAndTime = this.lastMarketSlotMap + .get(marketArgs.marketType) + .get(marketArgs.marketIndex); + if (!lastMarketSlotAndTime) { + this.lastMarketSlotMap + .get(marketArgs.marketType) + .set(marketArgs.marketIndex, { slot, ts: Date.now() }); + } + + if (slot) { + delete l3.slot; + } + const marketType = isVariant(marketArgs.marketType, 'perp') + ? 'perp' + : 'spot'; + + for (const key of Object.keys(l3)) { + for (const idx in l3[key]) { + const level = l3[key][idx]; + l3[key][idx] = { + ...level, + price: level.price.toString(), + size: level.size.toString(), + }; + } + } + + l3['marketName'] = marketName?.toUpperCase(); + l3['marketType'] = marketType?.toLowerCase(); + l3['marketIndex'] = marketArgs.marketIndex; + l3['ts'] = Date.now(); + l3['slot'] = slot; + addOracletoResponse( + l3, + this.driftClient, + marketArgs.marketType, + marketArgs.marketIndex + ); + addMarketSlotToResponse( + l3, + this.driftClient, + marketArgs.marketType, + marketArgs.marketIndex + ); + + // Check if slot diffs are too large for oracle + if ( + Math.abs(slot - parseInt(l3['oracleData']['slot'])) > + this.killSwitchSlotDiffThreshold + ) { + console.log(`Killing process due to slot diffs for market ${marketName}: + dlobProvider slot: ${slot} + oracle slot: ${l3['oracleData']['slot']} + `); + process.exit(1); + } + + // Check if times and slots are too different for market + const MAKRET_STALENESS_THRESHOLD = + marketType === 'perp' + ? PERP_MAKRET_STALENESS_THRESHOLD + : SPOT_MAKRET_STALENESS_THRESHOLD; + if ( + lastMarketSlotAndTime && + l3['marketSlot'] === lastMarketSlotAndTime.slot && + Date.now() - lastMarketSlotAndTime.ts > MAKRET_STALENESS_THRESHOLD + ) { + console.log(`Killing process due to same slot for market ${marketName} after > ${MAKRET_STALENESS_THRESHOLD}ms: + dlobProvider slot: ${slot} + market slot: ${l3['marketSlot']} + `); + process.exit(1); + } else if ( + lastMarketSlotAndTime && + l3['marketSlot'] !== lastMarketSlotAndTime.slot + ) { + console.log( + `Updating market slot for ${marketArgs.marketName} with slot ${l3['marketSlot']}` + ); + this.lastMarketSlotMap + .get(marketArgs.marketType) + .set(marketArgs.marketIndex, { + slot: l3['marketSlot'], + ts: Date.now(), + }); + } + + this.redisClient.client.set( + `last_update_orderbook_l3_${marketType}_${marketArgs.marketIndex}`, + JSON.stringify(l3) + ); + } } diff --git a/src/index.ts b/src/index.ts index 5092dd2..b5adc33 100644 --- a/src/index.ts +++ b/src/index.ts @@ -36,6 +36,7 @@ import { accountUpdatesCounter, cacheHitCounter, setLastReceivedWsMsgTs, + incomingRequestsCounter, runtimeSpecsGauge, } from './core/metrics'; import { handleResponseTime } from './core/middleware'; @@ -57,26 +58,29 @@ import { RedisClient } from './utils/redisClient'; require('dotenv').config(); // Reading in Redis env vars -const REDIS_HOSTS_ENV = (process.env.REDIS_HOSTS as string) || 'localhost'; -const REDIS_HOSTS = REDIS_HOSTS_ENV.includes(',') - ? REDIS_HOSTS_ENV.trim() - .replace(/^\[|\]$/g, '') - .split(/\s*,\s*/) - : [REDIS_HOSTS_ENV]; +const REDIS_HOSTS = process.env.REDIS_HOSTS?.replace(/^\[|\]$/g, '') + .split(',') + .map((host) => host.trim()) || ['localhost']; +const REDIS_PORTS = process.env.REDIS_PORTS?.replace(/^\[|\]$/g, '') + .split(',') + .map((port) => parseInt(port.trim(), 10)) || [6379]; +const REDIS_PASSWORDS_ENV = process.env.REDIS_PASSWORDS || "['']"; -const REDIS_PASSWORDS_ENV = (process.env.REDIS_PASSWORDS as string) || ''; -const REDIS_PASSWORDS = REDIS_PASSWORDS_ENV.includes(',') - ? REDIS_PASSWORDS_ENV.trim() - .replace(/^\[|\]$/g, '') - .split(/\s*,\s*/) - : [REDIS_PASSWORDS_ENV]; +let REDIS_PASSWORDS; + +if (REDIS_PASSWORDS_ENV.trim() === "['']") { + REDIS_PASSWORDS = [undefined]; +} else { + REDIS_PASSWORDS = REDIS_PASSWORDS_ENV.replace(/^\[|\]$/g, '') + .split(/\s*,\s*/) + .map((pwd) => pwd.replace(/(^'|'$)/g, '').trim()) + .map((pwd) => (pwd === '' ? undefined : pwd)); +} + +console.log('Redis Hosts:', REDIS_HOSTS); +console.log('Redis Ports:', REDIS_PORTS); +console.log('Redis Passwords:', REDIS_PASSWORDS); -const REDIS_PORTS_ENV = (process.env.REDIS_PORTS as string) || '6379'; -const REDIS_PORTS = REDIS_PORTS_ENV.includes(',') - ? REDIS_PORTS_ENV.trim() - .replace(/^\[|\]$/g, '') - .split(/\s*,\s*/) - : [REDIS_PORTS_ENV]; if ( REDIS_PORTS.length !== REDIS_PASSWORDS.length || REDIS_PORTS.length !== REDIS_HOSTS.length @@ -142,6 +146,7 @@ app.use((req, _res, next) => { req.url = '/'; } } + incomingRequestsCounter.add(1); next(); }); @@ -363,7 +368,7 @@ const main = async (): Promise => { redisClients.push( new RedisClient( REDIS_HOSTS[i], - REDIS_PORTS[i], + REDIS_PORTS[i].toString(), REDIS_PASSWORDS[i] || undefined ) ); @@ -1182,6 +1187,32 @@ const main = async (): Promise => { return; } + const marketTypeStr = getVariant(normedMarketType); + if (useRedis) { + const redisClient = ( + marketTypeStr === 'spot' ? spotMarketRedisMap : perpMarketRedisMap + ).get(normedMarketIndex).client; + const redisL3 = await redisClient.client.get( + `last_update_orderbook_l3_${marketTypeStr}_${normedMarketIndex}` + ); + if ( + redisL3 && + dlobProvider.getSlot() - parseInt(JSON.parse(redisL3).slot) < + SLOT_STALENESS_TOLERANCE + ) { + cacheHitCounter.add(1, { + miss: false, + }); + res.writeHead(200); + res.end(redisL3); + return; + } else { + cacheHitCounter.add(1, { + miss: true, + }); + } + } + const l3 = dlobSubscriber.getL3({ marketIndex: normedMarketIndex, marketType: normedMarketType,