From b318bc7a548f27889817bb3b3011b6a34b784685 Mon Sep 17 00:00:00 2001 From: wphan Date: Mon, 18 Dec 2023 15:36:59 -0800 Subject: [PATCH 1/3] dlob server self update --- src/dlobProvider.ts | 10 +++++++++ src/index.ts | 52 +++------------------------------------------ 2 files changed, 13 insertions(+), 49 deletions(-) diff --git a/src/dlobProvider.ts b/src/dlobProvider.ts index fc541c4..9bdb8c1 100644 --- a/src/dlobProvider.ts +++ b/src/dlobProvider.ts @@ -13,6 +13,7 @@ export type DLOBProvider = { getUserAccount(publicKey: PublicKey): UserAccount | undefined; size(): number; fetch(): Promise; + getSlot(): number; }; export function getDLOBProviderFromUserMap(userMap: UserMap): DLOBProvider { @@ -43,6 +44,9 @@ export function getDLOBProviderFromUserMap(userMap: UserMap): DLOBProvider { fetch: () => { return userMap.sync(); }, + getSlot: () => { + return userMap.getSlot(); + }, }; } @@ -82,6 +86,9 @@ export function getDLOBProviderFromOrderSubscriber( fetch() { return orderSubscriber.fetch(); }, + getSlot: () => { + return orderSubscriber.getSlot(); + }, }; } @@ -121,5 +128,8 @@ export function getDLOBProviderFromGrpcOrderSubscriber( fetch() { return orderSubscriber.fetch(); }, + getSlot: () => { + return orderSubscriber.getSlot(); + }, }; } diff --git a/src/index.ts b/src/index.ts index 1aeb7d0..e90d62a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,7 +20,6 @@ import { DriftEnv, SlotSource, SlotSubscriber, - UserMap, Wallet, getUserStatsAccountPublicKey, getVariant, @@ -34,7 +33,6 @@ import { logger, setLogLevel } from './utils/logger'; import * as http from 'http'; import { - gpaFetchDurationHistogram, handleHealthCheck, accountUpdatesCounter, cacheHitCounter, @@ -57,7 +55,6 @@ import FEATURE_FLAGS from './utils/featureFlags'; import { DLOBProvider, getDLOBProviderFromOrderSubscriber, - getDLOBProviderFromUserMap, } from './dlobProvider'; import { RedisClient } from './utils/redisClient'; @@ -249,10 +246,7 @@ const main = async () => { type: 'polling', accountLoader: bulkAccountLoader, }; - - slotSource = { - getSlot: () => bulkAccountLoader!.getSlot(), - }; + slotSource = bulkAccountLoader; } else { accountSubscription = { type: 'websocket', @@ -260,10 +254,6 @@ const main = async () => { }; slotSubscriber = new SlotSubscriber(connection); await slotSubscriber.subscribe(); - - slotSource = { - getSlot: () => slotSubscriber!.getSlot(), - }; } driftClient = new DriftClient({ @@ -281,6 +271,7 @@ const main = async () => { subscriptionConfig = { type: 'websocket', commitment: stateCommitment, + resyncIntervalMs: WS_FALLBACK_FETCH_INTERVAL, }; } else { subscriptionConfig = { @@ -306,23 +297,7 @@ const main = async () => { ); dlobProvider = getDLOBProviderFromOrderSubscriber(orderSubscriber); - - slotSource = { - getSlot: () => orderSubscriber.getSlot(), - }; - } else { - const userMap = new UserMap({ - driftClient, - subscriptionConfig: { - type: 'websocket', - resubTimeoutMs: 30_000, - commitment: stateCommitment, - }, - skipInitialLoad: false, - includeIdle: false, - }); - - dlobProvider = getDLOBProviderFromUserMap(userMap); + slotSource = dlobProvider; } const dlobCoder = DLOBOrdersCoder.create(); @@ -344,27 +319,6 @@ const main = async () => { logger.info( `GPA refresh?: ${useWebsocket && !FEATURE_FLAGS.DISABLE_GPA_REFRESH}` ); - if (useWebsocket && !FEATURE_FLAGS.DISABLE_GPA_REFRESH) { - const recursiveFetch = (delay = WS_FALLBACK_FETCH_INTERVAL) => { - setTimeout(() => { - const startFetch = Date.now(); - dlobProvider - .fetch() - .then(() => { - gpaFetchDurationHistogram.record(Date.now() - startFetch); - }) - .catch((e) => { - logger.error('Failed to fetch GPA'); - console.log(e); - }) - .finally(() => { - // eslint-disable-next-line @typescript-eslint/no-unused-vars - recursiveFetch(); - }); - }, delay); - }; - recursiveFetch(); - } logger.info(`Initializing DLOBSubscriber...`); const initDlobSubscriberStart = Date.now(); From 746ca73c5cd63625bd997ff716e9a6ff67a963e0 Mon Sep 17 00:00:00 2001 From: wphan Date: Mon, 18 Dec 2023 15:40:07 -0800 Subject: [PATCH 2/3] must use order subscriber --- src/index.ts | 99 +++++++++++++++++++++++----------------------------- 1 file changed, 43 insertions(+), 56 deletions(-) diff --git a/src/index.ts b/src/index.ts index e90d62a..aa7fc59 100644 --- a/src/index.ts +++ b/src/index.ts @@ -18,7 +18,6 @@ import { DriftClient, DriftClientSubscriptionConfig, DriftEnv, - SlotSource, SlotSubscriber, Wallet, getUserStatsAccountPublicKey, @@ -52,10 +51,7 @@ import { validateDlobQuery, } from './utils/utils'; import FEATURE_FLAGS from './utils/featureFlags'; -import { - DLOBProvider, - getDLOBProviderFromOrderSubscriber, -} from './dlobProvider'; +import { getDLOBProviderFromOrderSubscriber } from './dlobProvider'; import { RedisClient } from './utils/redisClient'; require('dotenv').config(); @@ -74,8 +70,6 @@ const serverPort = process.env.PORT || 6969; export const ORDERBOOK_UPDATE_INTERVAL = 1000; const WS_FALLBACK_FETCH_INTERVAL = ORDERBOOK_UPDATE_INTERVAL * 10; const useWebsocket = process.env.USE_WEBSOCKET?.toLowerCase() === 'true'; -const useOrderSubscriber = - process.env.USE_ORDER_SUBSCRIBER?.toLowerCase() === 'true'; const rateLimitCallsPerSecond = process.env.RATE_LIMIT_CALLS_PER_SECOND ? parseInt(process.env.RATE_LIMIT_CALLS_PER_SECOND) : 1; @@ -152,7 +146,6 @@ const wsEndpoint = process.env.WS_ENDPOINT; logger.info(`RPC endpoint: ${endpoint}`); logger.info(`WS endpoint: ${wsEndpoint}`); logger.info(`useWebsocket: ${useWebsocket}`); -logger.info(`useOrderSubscriber: ${useOrderSubscriber}`); logger.info(`DriftEnv: ${driftEnv}`); logger.info(`Commit: ${commitHash}`); @@ -233,7 +226,6 @@ const main = async () => { let slotSubscriber: SlotSubscriber | undefined; let accountSubscription: DriftClientSubscriptionConfig; - let slotSource: SlotSource; if (!useWebsocket) { bulkAccountLoader = new BulkAccountLoader( @@ -246,7 +238,6 @@ const main = async () => { type: 'polling', accountLoader: bulkAccountLoader, }; - slotSource = bulkAccountLoader; } else { accountSubscription = { type: 'websocket', @@ -264,42 +255,38 @@ const main = async () => { env: driftEnv, }); - let dlobProvider: DLOBProvider; - if (useOrderSubscriber) { - let subscriptionConfig; - if (useWebsocket) { - subscriptionConfig = { - type: 'websocket', - commitment: stateCommitment, - resyncIntervalMs: WS_FALLBACK_FETCH_INTERVAL, - }; - } else { - subscriptionConfig = { - type: 'polling', - frequency: ORDERBOOK_UPDATE_INTERVAL, - commitment: stateCommitment, - }; - } - - let updatesReceivedTotal = 0; - const orderSubscriber = new OrderSubscriber({ - driftClient, - subscriptionConfig, - }); - orderSubscriber.eventEmitter.on( - 'updateReceived', - (_pubkey: PublicKey, _slot: number, _dataType: 'raw' | 'decoded') => { - setLastReceivedWsMsgTs(Date.now()); - // eslint-disable-next-line @typescript-eslint/no-unused-vars - updatesReceivedTotal++; - accountUpdatesCounter.add(1); - } - ); - - dlobProvider = getDLOBProviderFromOrderSubscriber(orderSubscriber); - slotSource = dlobProvider; + let subscriptionConfig; + if (useWebsocket) { + subscriptionConfig = { + type: 'websocket', + commitment: stateCommitment, + resyncIntervalMs: WS_FALLBACK_FETCH_INTERVAL, + }; + } else { + subscriptionConfig = { + type: 'polling', + frequency: ORDERBOOK_UPDATE_INTERVAL, + commitment: stateCommitment, + }; } + let updatesReceivedTotal = 0; + const orderSubscriber = new OrderSubscriber({ + driftClient, + subscriptionConfig, + }); + orderSubscriber.eventEmitter.on( + 'updateReceived', + (_pubkey: PublicKey, _slot: number, _dataType: 'raw' | 'decoded') => { + setLastReceivedWsMsgTs(Date.now()); + // eslint-disable-next-line @typescript-eslint/no-unused-vars + updatesReceivedTotal++; + accountUpdatesCounter.add(1); + } + ); + + const dlobProvider = getDLOBProviderFromOrderSubscriber(orderSubscriber); + const dlobCoder = DLOBOrdersCoder.create(); await driftClient.subscribe(); @@ -325,7 +312,7 @@ const main = async () => { const dlobSubscriber = new DLOBSubscriber({ driftClient, dlobSource: dlobProvider, - slotSource, + slotSource: dlobProvider, updateFrequency: ORDERBOOK_UPDATE_INTERVAL, }); await dlobSubscriber.subscribe(); @@ -361,10 +348,10 @@ const main = async () => { app.get( '/health', - handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, slotSource) + handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, dlobProvider) ); app.get('/startup', handleStartup); - app.get('/', handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, slotSource)); + app.get('/', handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, dlobProvider)); if (FEATURE_FLAGS.ENABLE_ORDERS_ENDPOINTS) { app.get('/orders/json/raw', async (_req, res, next) => { @@ -372,7 +359,7 @@ const main = async () => { // object with userAccount key and orders object serialized const orders: Array = []; const oracles: Array = []; - const slot = slotSource.getSlot(); + const slot = dlobProvider.getSlot(); for (const market of driftClient.getPerpMarketAccounts()) { const oracle = driftClient.getOracleDataForPerpMarket( @@ -417,7 +404,7 @@ const main = async () => { app.get('/orders/json', async (_req, res, next) => { try { // object with userAccount key and orders object serialized - const slot = slotSource.getSlot(); + const slot = dlobProvider.getSlot(); const orders: Array = []; const oracles: Array = []; for (const market of driftClient.getPerpMarketAccounts()) { @@ -587,7 +574,7 @@ const main = async () => { res.end( JSON.stringify({ - slot: slotSource.getSlot(), + slot: dlobProvider.getSlot(), data: dlobCoder.encode(dlobOrders).toString('base64'), }) ); @@ -678,7 +665,7 @@ const main = async () => { .getDLOB() .getRestingLimitBids( normedMarketIndex, - slotSource.getSlot(), + dlobProvider.getSlot(), normedMarketType, oracle ) @@ -689,7 +676,7 @@ const main = async () => { .getDLOB() .getRestingLimitAsks( normedMarketIndex, - slotSource.getSlot(), + dlobProvider.getSlot(), normedMarketType, oracle ) @@ -763,7 +750,7 @@ const main = async () => { } if ( redisL2 && - slotSource.getSlot() - parseInt(JSON.parse(redisL2).slot) < 10 + dlobProvider.getSlot() - parseInt(JSON.parse(redisL2).slot) < 10 ) l2Formatted = redisL2; } else if ( @@ -789,7 +776,7 @@ const main = async () => { } if ( redisL2 && - slotSource.getSlot() - parseInt(JSON.parse(redisL2).slot) < 10 + dlobProvider.getSlot() - parseInt(JSON.parse(redisL2).slot) < 10 ) l2Formatted = redisL2; } @@ -945,7 +932,7 @@ const main = async () => { } if (redisL2) { const parsedRedisL2 = JSON.parse(redisL2); - if (slotSource.getSlot() - parseInt(parsedRedisL2.slot) < 10) + if (dlobProvider.getSlot() - parseInt(parsedRedisL2.slot) < 10) l2Formatted = parsedRedisL2; } } else if ( @@ -970,7 +957,7 @@ const main = async () => { } if (redisL2) { const parsedRedisL2 = JSON.parse(redisL2); - if (slotSource.getSlot() - parseInt(parsedRedisL2.slot) < 10) + if (dlobProvider.getSlot() - parseInt(parsedRedisL2.slot) < 10) l2Formatted = parsedRedisL2; } } From b1a805859dc2c9cfa32f66d96ea731fd31017e22 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 19 Dec 2023 17:53:24 +0000 Subject: [PATCH 3/3] Bumping sdk dependency to 2.52.0-beta.8 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index c892b90..7f2861c 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "license": "Apache-2.0", "dependencies": { "@coral-xyz/anchor": "^0.29.0", - "@drift-labs/sdk": "2.52.0-beta.7", + "@drift-labs/sdk": "2.52.0-beta.8", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index c8c1b4d..18a4fad 100644 --- a/yarn.lock +++ b/yarn.lock @@ -115,10 +115,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.52.0-beta.7": - version "2.52.0-beta.7" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.52.0-beta.7.tgz#9fb2cb24d140e0d66b27e6bd24ae2268a6d8a3b5" - integrity sha512-12I84O5EGU78fPWza+DMfqD23Zo2xMtsJbwyz6nrSKd840rGvI8KCVGnK7hv18dI/3uVmvdp79Mlg60LHieK0g== +"@drift-labs/sdk@2.52.0-beta.8": + version "2.52.0-beta.8" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.52.0-beta.8.tgz#87a44cb244affdf6c5a5e551cd16e33b9d38fd9c" + integrity sha512-r0qX2jrwXefw4SRPvJ0VqV1t61m9smmSdTjZcGFeYAj/zcROHfBh4htE1iCTO+E+yHocMELHZzHM1DkdP2aRDQ== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2"