From 6ef259b1e4b64ca1c9bbc0c938a92cc8ab92d170 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 22 Dec 2023 13:13:05 +0000 Subject: [PATCH 01/10] Bumping sdk dependency to 2.52.0-beta.15 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 066a4ee..9c1a5e7 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "license": "Apache-2.0", "dependencies": { "@coral-xyz/anchor": "^0.29.0", - "@drift-labs/sdk": "2.52.0-beta.14", + "@drift-labs/sdk": "2.52.0-beta.15", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 095f801..f496783 100644 --- a/yarn.lock +++ b/yarn.lock @@ -115,10 +115,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.52.0-beta.14": - version "2.52.0-beta.14" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.52.0-beta.14.tgz#93423eee5f3186df9d3a9a18c762c3b0693b6d3c" - integrity sha512-3lg8JuPBrWTlYPZr1JhRFWwYaTv92BArlYuATj9Hz4McMH70/PWAHgU2gZPXtzr97VIygGhAYjei0LH8j6yXhw== +"@drift-labs/sdk@2.52.0-beta.15": + version "2.52.0-beta.15" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.52.0-beta.15.tgz#6afb7e41ccd0c58f71e592f59bea4fc85a43278f" + integrity sha512-mlaV++pUbpSUgEAdSigSqbWKdVxkHlMvoFxazBzoScdLZZ7wlaqBZ8OfPXiwss3J7DL2c9MZGJltEsJXBuXpSQ== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From dc56d97d1421e9052c0adb6204445ab3254a647f Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 22 Dec 2023 21:11:29 +0000 Subject: [PATCH 02/10] Bumping sdk dependency to 2.53.0-beta.0 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 9c1a5e7..de0928d 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "license": "Apache-2.0", "dependencies": { "@coral-xyz/anchor": "^0.29.0", - "@drift-labs/sdk": "2.52.0-beta.15", + "@drift-labs/sdk": "2.53.0-beta.0", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index f496783..7b6f461 100644 --- a/yarn.lock +++ b/yarn.lock @@ -115,10 +115,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.52.0-beta.15": - version "2.52.0-beta.15" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.52.0-beta.15.tgz#6afb7e41ccd0c58f71e592f59bea4fc85a43278f" - integrity sha512-mlaV++pUbpSUgEAdSigSqbWKdVxkHlMvoFxazBzoScdLZZ7wlaqBZ8OfPXiwss3J7DL2c9MZGJltEsJXBuXpSQ== +"@drift-labs/sdk@2.53.0-beta.0": + version "2.53.0-beta.0" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.53.0-beta.0.tgz#b0bc0449d8af1e227e254da5fa9d58b4a388fc0d" + integrity sha512-JkujfAncR4MDhfise9qSPe6X+KtO7rKB0Uh1DLmuEb2mHEJigoTT5vCwzQ0cpXvw+nuLE5DIjGp+7WlNfjx3zw== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From e53b33d062f52b6236696741cfc64efc0bc68fd0 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 26 Dec 2023 18:12:16 +0000 Subject: [PATCH 03/10] Bumping sdk dependency to 2.53.0-beta.1 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index de0928d..3fda67d 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "license": "Apache-2.0", "dependencies": { "@coral-xyz/anchor": "^0.29.0", - "@drift-labs/sdk": "2.53.0-beta.0", + "@drift-labs/sdk": "2.53.0-beta.1", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 7b6f461..f3d93f4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -115,10 +115,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.53.0-beta.0": - version "2.53.0-beta.0" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.53.0-beta.0.tgz#b0bc0449d8af1e227e254da5fa9d58b4a388fc0d" - integrity sha512-JkujfAncR4MDhfise9qSPe6X+KtO7rKB0Uh1DLmuEb2mHEJigoTT5vCwzQ0cpXvw+nuLE5DIjGp+7WlNfjx3zw== +"@drift-labs/sdk@2.53.0-beta.1": + version "2.53.0-beta.1" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.53.0-beta.1.tgz#91671fc7d6550ecdd2f3b9a5cc82ec358e033fd6" + integrity sha512-pzqnFykg0hDwViP5N+i7mKvqHDEwLbfRPRTLzV/gvE8rfcuDQLQm3Ce33UwO1I3ufkvQ97PGgGUJ9l2SrqcB5A== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From b94b100c77a0422b7f82945477a06d8969fba579 Mon Sep 17 00:00:00 2001 From: wphan Date: Sat, 23 Dec 2023 15:11:52 -0800 Subject: [PATCH 04/10] take in perp and spot markets from env var --- src/publishers/dlobPublisher.ts | 101 +++++++++++++++++++++++++++++--- 1 file changed, 94 insertions(+), 7 deletions(-) diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index a036b1d..9858379 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -14,6 +14,8 @@ import { DriftClientSubscriptionConfig, SlotSubscriber, isVariant, + OracleInfo, + PerpMarketConfig, } from '@drift-labs/sdk'; import { logger, setLogLevel } from '../utils/logger'; @@ -64,6 +66,16 @@ const ORDERBOOK_UPDATE_INTERVAL = parseInt(process.env.ORDERBOOK_UPDATE_INTERVAL) || 1000; const WS_FALLBACK_FETCH_INTERVAL = 10_000; +// comma separated list of perp market indexes to load: i.e. 0,1,2,3 +const PERP_MARKETS_TO_LOAD = process.env.PERP_MARKETS_TO_LOAD + ? process.env.PERP_MARKETS_TO_LOAD.split(',').map((m) => parseInt(m)) + : undefined; + +// comma separated list of spot market indexes to load: i.e. 0,1,2,3 +const SPOT_MARKETS_TO_LOAD = process.env.SPOT_MARKETS_TO_LOAD + ? process.env.SPOT_MARKETS_TO_LOAD.split(',').map((m) => parseInt(m)) + : undefined; + logger.info(`RPC endpoint: ${endpoint}`); logger.info(`WS endpoint: ${wsEndpoint}`); logger.info(`DriftEnv: ${driftEnv}`); @@ -71,22 +83,89 @@ logger.info(`Commit: ${commitHash}`); let MARKET_SUBSCRIBERS: SubscriberLookup = {}; +const getMarketsAndOraclesToLoad = ( + sdkConfig: any +): { + perpMarketIndexes?: number[]; + spotMarketIndexes?: number[]; + oracleInfos?: OracleInfo[]; +} => { + const oracleInfos: OracleInfo[] = []; + const oraclesTracked = new Set(); + let perpMarketIndexes: number[] = []; + let spotMarketIndexes: number[] = []; + + if (PERP_MARKETS_TO_LOAD!.length > 0) { + perpMarketIndexes = PERP_MARKETS_TO_LOAD; + logger.info(`DlobPublisher tracking perp markets: ${perpMarketIndexes}`); + + for (const idx of perpMarketIndexes) { + const perpMarketConfig = sdkConfig.PERP_MARKETS[idx] as PerpMarketConfig; + if (!perpMarketConfig) { + throw new Error(`Perp market config for ${idx} not found`); + } + const oracleKey = perpMarketConfig.oracle.toBase58(); + if (!oraclesTracked.has(oracleKey)) { + logger.info(`Tracking oracle ${oracleKey} for perp market ${idx}`); + oracleInfos.push({ + publicKey: perpMarketConfig.oracle, + source: perpMarketConfig.oracleSource, + }); + oraclesTracked.add(oracleKey); + } + } + } + + if (SPOT_MARKETS_TO_LOAD!.length > 0) { + spotMarketIndexes = SPOT_MARKETS_TO_LOAD; + logger.info(`DlobPublisher tracking spot markets: ${spotMarketIndexes}`); + + for (const idx of spotMarketIndexes) { + const spotMarketConfig = sdkConfig.PERP_MARKETS[idx] as PerpMarketConfig; + if (!spotMarketConfig) { + throw new Error(`Spot market config for ${idx} not found`); + } + const oracleKey = spotMarketConfig.oracle.toBase58(); + if (!oraclesTracked.has(oracleKey)) { + logger.info(`Tracking oracle ${oracleKey} for spot market ${idx}`); + oracleInfos.push({ + publicKey: spotMarketConfig.oracle, + source: spotMarketConfig.oracleSource, + }); + oraclesTracked.add(oracleKey); + } + } + } + + return { + perpMarketIndexes, + spotMarketIndexes, + oracleInfos, + }; +}; + const initializeAllMarketSubscribers = async (driftClient: DriftClient) => { const markets: SubscriberLookup = {}; - for (const market of sdkConfig.SPOT_MARKETS) { + for (const market of driftClient.getSpotMarketAccounts()) { markets[market.marketIndex] = { phoenix: undefined, serum: undefined, }; + const marketConfig = sdkConfig.SPOT_MARKETS[market.marketIndex]; - if (market.phoenixMarket) { + if (marketConfig.phoenixMarket) { const phoenixConfigAccount = - await driftClient.getPhoenixV1FulfillmentConfig(market.phoenixMarket); + await driftClient.getPhoenixV1FulfillmentConfig( + marketConfig.phoenixMarket + ); if (isVariant(phoenixConfigAccount.status, 'enabled')) { + logger.info( + `Loading phoenix subscriber for spot market ${market.marketIndex}` + ); const phoenixSubscriber = getPhoenixSubscriber( driftClient, - market, + marketConfig, sdkConfig ); await phoenixSubscriber.subscribe(); @@ -103,14 +182,17 @@ const initializeAllMarketSubscribers = async (driftClient: DriftClient) => { } } - if (market.serumMarket) { + if (marketConfig.serumMarket) { const serumConfigAccount = await driftClient.getSerumV3FulfillmentConfig( - market.serumMarket + marketConfig.serumMarket ); if (isVariant(serumConfigAccount.status, 'enabled')) { + logger.info( + `Loading serum subscriber for spot market ${market.marketIndex}` + ); const serumSubscriber = getSerumSubscriber( driftClient, - market, + marketConfig, sdkConfig ); await serumSubscriber.subscribe(); @@ -177,12 +259,17 @@ const main = async () => { }; } + const { perpMarketIndexes, spotMarketIndexes, oracleInfos } = + getMarketsAndOraclesToLoad(sdkConfig); driftClient = new DriftClient({ connection, wallet, programID: clearingHousePublicKey, accountSubscription, env: driftEnv, + perpMarketIndexes, + spotMarketIndexes, + oracleInfos, }); const lamportsBalance = await connection.getBalance(wallet.publicKey); From 95b99b2ffa353a0701b2363bba82c0dc73dc4183 Mon Sep 17 00:00:00 2001 From: wphan Date: Sun, 24 Dec 2023 01:35:17 -0800 Subject: [PATCH 05/10] pass list of markets into DLOBSubscriberIO --- src/dlob-subscriber/DLOBSubscriberIO.ts | 26 ++++++------- src/publishers/dlobPublisher.ts | 49 ++++++++++++++++--------- 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 7dd331f..8b88c23 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -2,16 +2,11 @@ import { BN, DLOBSubscriber, DLOBSubscriptionConfig, - DevnetPerpMarkets, - DevnetSpotMarkets, L2OrderBookGenerator, - MainnetPerpMarkets, - MainnetSpotMarkets, MarketType, groupL2, isVariant, } from '@drift-labs/sdk'; -import { driftEnv } from '../publishers/dlobPublisher'; import { RedisClient } from '../utils/redisClient'; import { SubscriberLookup, @@ -33,6 +28,11 @@ type wsMarketL2Args = { updateOnChange?: boolean; }; +export type wsMarketInfo = { + marketIndex: number; + marketName: string; +}; + export class DLOBSubscriberIO extends DLOBSubscriber { public marketL2Args: wsMarketL2Args[] = []; public lastSeenL2Formatted: Map>; @@ -41,6 +41,8 @@ export class DLOBSubscriberIO extends DLOBSubscriber { constructor( config: DLOBSubscriptionConfig & { redisClient: RedisClient; + perpMarketsInfos: wsMarketInfo[]; + spotMarketsInfos: wsMarketInfo[]; spotMarketSubscribers: SubscriberLookup; } ) { @@ -52,17 +54,11 @@ export class DLOBSubscriberIO extends DLOBSubscriber { this.lastSeenL2Formatted.set(MarketType.SPOT, new Map()); this.lastSeenL2Formatted.set(MarketType.PERP, new Map()); - // Add all active markets to the market L2Args - const perpMarkets = - driftEnv === 'devnet' ? DevnetPerpMarkets : MainnetPerpMarkets; - const spotMarkets = - driftEnv === 'devnet' ? DevnetSpotMarkets : MainnetSpotMarkets; - - for (const market of perpMarkets) { + for (const market of config.perpMarketsInfos) { this.marketL2Args.push({ marketIndex: market.marketIndex, marketType: MarketType.PERP, - marketName: market.symbol, + marketName: market.marketName, depth: -1, numVammOrders: 100, includeVamm: true, @@ -70,11 +66,11 @@ export class DLOBSubscriberIO extends DLOBSubscriber { fallbackL2Generators: [], }); } - for (const market of spotMarkets) { + for (const market of config.spotMarketsInfos) { this.marketL2Args.push({ marketIndex: market.marketIndex, marketType: MarketType.SPOT, - marketName: market.symbol, + marketName: market.marketName, depth: -1, includeVamm: false, updateOnChange: true, diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 9858379..f3063d6 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -25,7 +25,10 @@ import { getSerumSubscriber, sleep, } from '../utils/utils'; -import { DLOBSubscriberIO } from '../dlob-subscriber/DLOBSubscriberIO'; +import { + DLOBSubscriberIO, + wsMarketInfo, +} from '../dlob-subscriber/DLOBSubscriberIO'; import { RedisClient } from '../utils/redisClient'; import { DLOBProvider, @@ -86,20 +89,17 @@ let MARKET_SUBSCRIBERS: SubscriberLookup = {}; const getMarketsAndOraclesToLoad = ( sdkConfig: any ): { - perpMarketIndexes?: number[]; - spotMarketIndexes?: number[]; + perpMarketInfos: wsMarketInfo[]; + spotMarketInfos: wsMarketInfo[]; oracleInfos?: OracleInfo[]; } => { const oracleInfos: OracleInfo[] = []; const oraclesTracked = new Set(); - let perpMarketIndexes: number[] = []; - let spotMarketIndexes: number[] = []; + const perpMarketInfos: wsMarketInfo[] = []; + const spotMarketInfos: wsMarketInfo[] = []; if (PERP_MARKETS_TO_LOAD!.length > 0) { - perpMarketIndexes = PERP_MARKETS_TO_LOAD; - logger.info(`DlobPublisher tracking perp markets: ${perpMarketIndexes}`); - - for (const idx of perpMarketIndexes) { + for (const idx of PERP_MARKETS_TO_LOAD) { const perpMarketConfig = sdkConfig.PERP_MARKETS[idx] as PerpMarketConfig; if (!perpMarketConfig) { throw new Error(`Perp market config for ${idx} not found`); @@ -113,14 +113,18 @@ const getMarketsAndOraclesToLoad = ( }); oraclesTracked.add(oracleKey); } + perpMarketInfos.push({ + marketIndex: perpMarketConfig.marketIndex, + marketName: perpMarketConfig.symbol, + }); } + logger.info( + `DlobPublisher tracking perp markets: ${JSON.stringify(perpMarketInfos)}` + ); } if (SPOT_MARKETS_TO_LOAD!.length > 0) { - spotMarketIndexes = SPOT_MARKETS_TO_LOAD; - logger.info(`DlobPublisher tracking spot markets: ${spotMarketIndexes}`); - - for (const idx of spotMarketIndexes) { + for (const idx of SPOT_MARKETS_TO_LOAD) { const spotMarketConfig = sdkConfig.PERP_MARKETS[idx] as PerpMarketConfig; if (!spotMarketConfig) { throw new Error(`Spot market config for ${idx} not found`); @@ -134,12 +138,19 @@ const getMarketsAndOraclesToLoad = ( }); oraclesTracked.add(oracleKey); } + spotMarketInfos.push({ + marketIndex: spotMarketConfig.marketIndex, + marketName: spotMarketConfig.symbol, + }); } + logger.info( + `DlobPublisher tracking spot markets: ${JSON.stringify(spotMarketInfos)}` + ); } return { - perpMarketIndexes, - spotMarketIndexes, + perpMarketInfos, + spotMarketInfos, oracleInfos, }; }; @@ -259,7 +270,7 @@ const main = async () => { }; } - const { perpMarketIndexes, spotMarketIndexes, oracleInfos } = + const { perpMarketInfos, spotMarketInfos, oracleInfos } = getMarketsAndOraclesToLoad(sdkConfig); driftClient = new DriftClient({ connection, @@ -267,8 +278,8 @@ const main = async () => { programID: clearingHousePublicKey, accountSubscription, env: driftEnv, - perpMarketIndexes, - spotMarketIndexes, + perpMarketIndexes: perpMarketInfos.map((m) => m.marketIndex), + spotMarketIndexes: spotMarketInfos.map((m) => m.marketIndex), oracleInfos, }); @@ -358,6 +369,8 @@ const main = async () => { updateFrequency: ORDERBOOK_UPDATE_INTERVAL, redisClient, spotMarketSubscribers: MARKET_SUBSCRIBERS, + perpMarketsInfos, + spotMarketsInfos, }); await dlobSubscriber.subscribe(); if (useWebsocket && !FEATURE_FLAGS.DISABLE_GPA_REFRESH) { From 1dec1962b9638109a2f4961bcc804231c302fea5 Mon Sep 17 00:00:00 2001 From: wphan Date: Sun, 24 Dec 2023 01:58:43 -0800 Subject: [PATCH 06/10] tested --- src/dlob-subscriber/DLOBSubscriberIO.ts | 10 +++++----- src/publishers/dlobPublisher.ts | 21 ++++++++++++++------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 8b88c23..1494b52 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -41,8 +41,8 @@ export class DLOBSubscriberIO extends DLOBSubscriber { constructor( config: DLOBSubscriptionConfig & { redisClient: RedisClient; - perpMarketsInfos: wsMarketInfo[]; - spotMarketsInfos: wsMarketInfo[]; + perpMarketInfos: wsMarketInfo[]; + spotMarketInfos: wsMarketInfo[]; spotMarketSubscribers: SubscriberLookup; } ) { @@ -54,7 +54,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { this.lastSeenL2Formatted.set(MarketType.SPOT, new Map()); this.lastSeenL2Formatted.set(MarketType.PERP, new Map()); - for (const market of config.perpMarketsInfos) { + for (const market of config.perpMarketInfos) { this.marketL2Args.push({ marketIndex: market.marketIndex, marketType: MarketType.PERP, @@ -66,7 +66,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { fallbackL2Generators: [], }); } - for (const market of config.spotMarketsInfos) { + for (const market of config.spotMarketInfos) { this.marketL2Args.push({ marketIndex: market.marketIndex, marketType: MarketType.SPOT, @@ -88,7 +88,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { try { this.getL2AndSendMsg(l2Args); } catch (error) { - logger.error(error); + console.error(error); console.log(`Error getting L2 ${l2Args.marketName}`); } } diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index f3063d6..450bad9 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -16,6 +16,7 @@ import { isVariant, OracleInfo, PerpMarketConfig, + SpotMarketConfig, } from '@drift-labs/sdk'; import { logger, setLogLevel } from '../utils/logger'; @@ -98,8 +99,11 @@ const getMarketsAndOraclesToLoad = ( const perpMarketInfos: wsMarketInfo[] = []; const spotMarketInfos: wsMarketInfo[] = []; - if (PERP_MARKETS_TO_LOAD!.length > 0) { - for (const idx of PERP_MARKETS_TO_LOAD) { + const perpIndexes = PERP_MARKETS_TO_LOAD + ? PERP_MARKETS_TO_LOAD + : sdkConfig.PERP_MARKETS.map((m) => m.marketIndex); + if (perpIndexes.length > 0) { + for (const idx of perpIndexes) { const perpMarketConfig = sdkConfig.PERP_MARKETS[idx] as PerpMarketConfig; if (!perpMarketConfig) { throw new Error(`Perp market config for ${idx} not found`); @@ -123,9 +127,12 @@ const getMarketsAndOraclesToLoad = ( ); } - if (SPOT_MARKETS_TO_LOAD!.length > 0) { - for (const idx of SPOT_MARKETS_TO_LOAD) { - const spotMarketConfig = sdkConfig.PERP_MARKETS[idx] as PerpMarketConfig; + const spotIndexes = SPOT_MARKETS_TO_LOAD + ? SPOT_MARKETS_TO_LOAD + : sdkConfig.SPOT_MARKETS.map((m) => m.marketIndex); + if (spotIndexes.length > 0) { + for (const idx of spotIndexes) { + const spotMarketConfig = sdkConfig.SPOT_MARKETS[idx] as SpotMarketConfig; if (!spotMarketConfig) { throw new Error(`Spot market config for ${idx} not found`); } @@ -369,8 +376,8 @@ const main = async () => { updateFrequency: ORDERBOOK_UPDATE_INTERVAL, redisClient, spotMarketSubscribers: MARKET_SUBSCRIBERS, - perpMarketsInfos, - spotMarketsInfos, + perpMarketInfos, + spotMarketInfos, }); await dlobSubscriber.subscribe(); if (useWebsocket && !FEATURE_FLAGS.DISABLE_GPA_REFRESH) { From 53ddc425403e9a533db6d3d3cb55f03ece7c91e5 Mon Sep 17 00:00:00 2001 From: wphan Date: Sun, 24 Dec 2023 02:03:53 -0800 Subject: [PATCH 07/10] only default to all markets if neither env vars are specified --- src/publishers/dlobPublisher.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 450bad9..dc1594b 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -99,9 +99,22 @@ const getMarketsAndOraclesToLoad = ( const perpMarketInfos: wsMarketInfo[] = []; const spotMarketInfos: wsMarketInfo[] = []; - const perpIndexes = PERP_MARKETS_TO_LOAD - ? PERP_MARKETS_TO_LOAD - : sdkConfig.PERP_MARKETS.map((m) => m.marketIndex); + // only watch all markets if neither env vars are specified + const noMarketsSpecified = !PERP_MARKETS_TO_LOAD && !SPOT_MARKETS_TO_LOAD; + + let perpIndexes = PERP_MARKETS_TO_LOAD; + if (!perpIndexes && noMarketsSpecified) { + perpIndexes = sdkConfig.PERP_MARKETS.map((m) => m.marketIndex); + } else { + perpIndexes = []; + } + let spotIndexes = SPOT_MARKETS_TO_LOAD; + if (!spotIndexes && noMarketsSpecified) { + spotIndexes = sdkConfig.SPOT_MARKETS.map((m) => m.marketIndex); + } else { + spotIndexes = []; + } + if (perpIndexes.length > 0) { for (const idx of perpIndexes) { const perpMarketConfig = sdkConfig.PERP_MARKETS[idx] as PerpMarketConfig; @@ -127,9 +140,6 @@ const getMarketsAndOraclesToLoad = ( ); } - const spotIndexes = SPOT_MARKETS_TO_LOAD - ? SPOT_MARKETS_TO_LOAD - : sdkConfig.SPOT_MARKETS.map((m) => m.marketIndex); if (spotIndexes.length > 0) { for (const idx of spotIndexes) { const spotMarketConfig = sdkConfig.SPOT_MARKETS[idx] as SpotMarketConfig; From 137f9343410d51a7e2922efeace95b60b4fd462c Mon Sep 17 00:00:00 2001 From: wphan Date: Tue, 26 Dec 2023 10:21:16 -0800 Subject: [PATCH 08/10] fix no markets specified behavior --- src/publishers/dlobPublisher.ts | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index dc1594b..479ac26 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -103,16 +103,20 @@ const getMarketsAndOraclesToLoad = ( const noMarketsSpecified = !PERP_MARKETS_TO_LOAD && !SPOT_MARKETS_TO_LOAD; let perpIndexes = PERP_MARKETS_TO_LOAD; - if (!perpIndexes && noMarketsSpecified) { - perpIndexes = sdkConfig.PERP_MARKETS.map((m) => m.marketIndex); - } else { - perpIndexes = []; + if (!perpIndexes) { + if (noMarketsSpecified) { + perpIndexes = sdkConfig.PERP_MARKETS.map((m) => m.marketIndex); + } else { + perpIndexes = []; + } } let spotIndexes = SPOT_MARKETS_TO_LOAD; - if (!spotIndexes && noMarketsSpecified) { - spotIndexes = sdkConfig.SPOT_MARKETS.map((m) => m.marketIndex); - } else { - spotIndexes = []; + if (!spotIndexes) { + if (noMarketsSpecified) { + spotIndexes = sdkConfig.SPOT_MARKETS.map((m) => m.marketIndex); + } else { + spotIndexes = []; + } } if (perpIndexes.length > 0) { From dac60ff9672bcd95a3c1d2b2ceb11514a0b5ef56 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Tue, 26 Dec 2023 10:40:37 -0800 Subject: [PATCH 09/10] better env var parser --- src/publishers/dlobPublisher.ts | 5 +++-- src/utils/utils.ts | 11 +++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 479ac26..9b01bdb 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -24,6 +24,7 @@ import { SubscriberLookup, getPhoenixSubscriber, getSerumSubscriber, + parsePositiveIntArray, sleep, } from '../utils/utils'; import { @@ -72,12 +73,12 @@ const WS_FALLBACK_FETCH_INTERVAL = 10_000; // comma separated list of perp market indexes to load: i.e. 0,1,2,3 const PERP_MARKETS_TO_LOAD = process.env.PERP_MARKETS_TO_LOAD - ? process.env.PERP_MARKETS_TO_LOAD.split(',').map((m) => parseInt(m)) + ? parsePositiveIntArray(process.env.PERP_MARKETS_TO_LOAD) : undefined; // comma separated list of spot market indexes to load: i.e. 0,1,2,3 const SPOT_MARKETS_TO_LOAD = process.env.SPOT_MARKETS_TO_LOAD - ? process.env.SPOT_MARKETS_TO_LOAD.split(',').map((m) => parseInt(m)) + ? parsePositiveIntArray(process.env.SPOT_MARKETS_TO_LOAD) : undefined; logger.info(`RPC endpoint: ${endpoint}`); diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 7d6e966..5f9db01 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -39,6 +39,17 @@ export function sleep(ms: number): Promise { return new Promise((resolve) => setTimeout(resolve, ms)); } +export function parsePositiveIntArray( + intArray: string, + separator = ',' +): number[] { + return intArray + .split(separator) + .map((s) => s.trim()) + .map((s) => parseInt(s)) + .filter((n) => !isNaN(n) && n > 0); +} + export const getOracleForMarket = ( driftClient: DriftClient, marketType: MarketType, From 86fcb9b8b4624e751bbd0e46ecde806b65fc3097 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Mon, 8 Jan 2024 12:30:05 -0800 Subject: [PATCH 10/10] permessagedeflate --- src/wsConnectionManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wsConnectionManager.ts b/src/wsConnectionManager.ts index db6d8a2..836f6dd 100644 --- a/src/wsConnectionManager.ts +++ b/src/wsConnectionManager.ts @@ -26,7 +26,7 @@ const server = http.createServer(app); const wss = new WebSocketServer({ server, path: '/ws', - perMessageDeflate: false, + perMessageDeflate: true, }); const REDIS_HOST = process.env.REDIS_HOST || 'localhost';