From 29f12d76f928a2ce0d49375b41934265f5661364 Mon Sep 17 00:00:00 2001 From: wphan Date: Tue, 28 Nov 2023 13:36:24 -0800 Subject: [PATCH 01/12] sync usermap on startup, update adhoc after --- src/index.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/index.ts b/src/index.ts index e327da4..2a76e16 100644 --- a/src/index.ts +++ b/src/index.ts @@ -238,15 +238,17 @@ const main = async () => { const userMap = new UserMap( driftClient, driftClient.userAccountSubscriptionConfig, - false, - async (authorities) => { - await userStatsMap.sync(authorities); - }, - { hasOpenOrders: true } + false ); await userMap.subscribe(); logger.info(`userMap initialized in ${Date.now() - initUserMapStart} ms`); + const initUserStatsMapStarts = Date.now(); + await userStatsMap.sync(userMap.getUniqueAuthorities()); + logger.info( + `userStatsMap initialized in ${Date.now() - initUserStatsMapStarts} ms` + ); + logger.info(`Initializing DLOBSubscriber...`); const initDlobSubscriberStart = Date.now(); const dlobSubscriber = new DLOBSubscriber({ From acf753fd8f202b752ba12062ce5e6437f0c69c0c Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 21:40:22 +0000 Subject: [PATCH 02/12] Bumping sdk dependency to 2.48.0-beta.10 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 8e7e9ba..e6f9255 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.48.0-beta.9", + "@drift-labs/sdk": "2.48.0-beta.10", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index fe86aa4..f0588b2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.48.0-beta.9": - version "2.48.0-beta.9" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.48.0-beta.9.tgz#2d0c99f58387ff3c5e45859ae4fe088339c8dd55" - integrity sha512-y0wrq680nv7Uwvcg9jOrjZPXxoae9yRg8M9B4bQgDFeNJmZNmcEmnLhWQ+CbPi8u/O0glhOdtKUk4UF9eSPXAg== +"@drift-labs/sdk@2.48.0-beta.10": + version "2.48.0-beta.10" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.48.0-beta.10.tgz#c97e88216159da34f687762b40cab3ce971c6d39" + integrity sha512-g2Zxdg/BrgmnMtI46uz4idc2M0jC+Vsz03cmxMEfYuu5JwyMjgOjgU4WyuBDPtLV8ovbrgJ2GYPiOO5OpkQeJg== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From d5c2bc1b013ad3ff0d81923c64ee21c58de4fed9 Mon Sep 17 00:00:00 2001 From: wphan Date: Tue, 28 Nov 2023 15:01:07 -0800 Subject: [PATCH 03/12] remove access logging --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 2a76e16..8caed26 100644 --- a/src/index.ts +++ b/src/index.ts @@ -66,7 +66,7 @@ const loadTestAllowed = process.env.ALLOW_LOAD_TEST?.toLowerCase() === 'true'; const logFormat = ':remote-addr - :remote-user [:date[clf]] ":method :url HTTP/:http-version" :status :res[content-length] ":referrer" ":user-agent" :req[x-forwarded-for]'; const logHttp = morgan(logFormat, { - skip: (_req, res) => res.statusCode < 400, + skip: (_req, res) => res.statusCode <= 500, }); let driftClient: DriftClient; From 96dc728330f155fc9c61093869012b2f98b82662 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 28 Nov 2023 23:07:02 +0000 Subject: [PATCH 04/12] Bumping sdk dependency to 2.48.0-beta.11 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index e6f9255..9b8a8f8 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.48.0-beta.10", + "@drift-labs/sdk": "2.48.0-beta.11", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index f0588b2..09455f5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.48.0-beta.10": - version "2.48.0-beta.10" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.48.0-beta.10.tgz#c97e88216159da34f687762b40cab3ce971c6d39" - integrity sha512-g2Zxdg/BrgmnMtI46uz4idc2M0jC+Vsz03cmxMEfYuu5JwyMjgOjgU4WyuBDPtLV8ovbrgJ2GYPiOO5OpkQeJg== +"@drift-labs/sdk@2.48.0-beta.11": + version "2.48.0-beta.11" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.48.0-beta.11.tgz#cd2f7b908754d49f62964ab894d3ab6c248dee3f" + integrity sha512-lhBlBUmKT3w9EMTDIBcEUdRgqZ6OHLfqPukxW4gVoa6eFqIpcaTk/DGTj3HCli6C684YwVHJfCQx4kXfffv6Sw== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 9baff8226228b6e7909ca64f1ae68c4ba8ced4fa Mon Sep 17 00:00:00 2001 From: Chris Heaney Date: Tue, 28 Nov 2023 19:53:36 -0500 Subject: [PATCH 05/12] init order subscriber flag --- src/dlobProvider.ts | 65 ++++++++++++++++++++++++++++++++++ src/index.ts | 85 +++++++++++++++++++++++++++++---------------- 2 files changed, 120 insertions(+), 30 deletions(-) create mode 100644 src/dlobProvider.ts diff --git a/src/dlobProvider.ts b/src/dlobProvider.ts new file mode 100644 index 0000000..bfa58f2 --- /dev/null +++ b/src/dlobProvider.ts @@ -0,0 +1,65 @@ +import {DLOB, OrderSubscriber, UserAccount, UserMap} from "@drift-labs/sdk"; +import {PublicKey} from "@solana/web3.js"; + +export type DLOBProvider = { + subscribe() : Promise; + getDLOB(slot: number): Promise; + getUniqueAuthorities(): PublicKey[]; + getUserAccounts(): Generator<{userAccount: UserAccount, publicKey: PublicKey}>; + getUserAccount(publicKey: PublicKey): UserAccount | undefined; + size(): number; +} + +export function getDLOBProviderFromUserMap(userMap:UserMap) : DLOBProvider { + return { + subscribe: async () => { + await userMap.subscribe(); + }, + getDLOB: async (slot: number) => { + return await userMap.getDLOB(slot); + }, + getUniqueAuthorities: () => { + return userMap.getUniqueAuthorities(); + }, + getUserAccounts: function*() { + for (const user of userMap.values()) { + yield {userAccount: user.getUserAccount(), publicKey: user.getUserAccountPublicKey()}; + } + }, + getUserAccount: (publicKey) => { + return userMap.get(publicKey.toString())?.getUserAccount(); + }, + size: () => { + return userMap.size(); + } + }; +} + +export function getDLOBProviderFromOrderSubscriber(orderSubscriber: OrderSubscriber) : DLOBProvider { + return { + subscribe: async () => { + await orderSubscriber.subscribe(); + }, + getDLOB: async (slot: number) => { + return await orderSubscriber.getDLOB(slot); + }, + getUniqueAuthorities: () => { + const authorities = new Set(); + for (const { userAccount } of orderSubscriber.usersAccounts.values()) { + authorities.add(userAccount.authority); + } + return Array.from(authorities.values()); + }, + getUserAccounts: function*() { + for (const [key, {userAccount}] of orderSubscriber.usersAccounts.entries()) { + yield {userAccount: userAccount, publicKey: new PublicKey(key)}; + } + }, + getUserAccount: (publicKey) => { + return orderSubscriber.usersAccounts.get(publicKey.toString())?.userAccount; + }, + size(): number { + return orderSubscriber.usersAccounts.size; + } + }; +} diff --git a/src/index.ts b/src/index.ts index 8caed26..f96be35 100644 --- a/src/index.ts +++ b/src/index.ts @@ -27,6 +27,7 @@ import { groupL2, initialize, isVariant, + OrderSubscriber, } from '@drift-labs/sdk'; import { logger, setLogLevel } from './utils/logger'; @@ -46,6 +47,7 @@ import { sleep, validateDlobQuery, } from './utils/utils'; +import {DLOBProvider, getDLOBProviderFromOrderSubscriber, getDLOBProviderFromUserMap} from "./dlobProvider"; require('dotenv').config(); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; @@ -57,6 +59,7 @@ const stateCommitment: Commitment = 'processed'; const serverPort = process.env.PORT || 6969; export const ORDERBOOK_UPDATE_INTERVAL = 1000; const useWebsocket = process.env.USE_WEBSOCKET?.toLowerCase() === 'true'; +const useOrderSubscriber = process.env.USE_ORDER_SUBSCRIBER?.toLowerCase() === 'true'; const rateLimitCallsPerSecond = process.env.RATE_LIMIT_CALLS_PER_SECOND ? parseInt(process.env.RATE_LIMIT_CALLS_PER_SECOND) @@ -196,6 +199,7 @@ const main = async () => { type: 'polling', accountLoader: bulkAccountLoader, }; + slotSource = { getSlot: () => bulkAccountLoader!.getSlot(), }; @@ -206,6 +210,7 @@ const main = async () => { }; slotSubscriber = new SlotSubscriber(connection); await slotSubscriber.subscribe(); + slotSource = { getSlot: () => slotSubscriber!.getSlot(), }; @@ -219,6 +224,40 @@ const main = async () => { env: driftEnv, }); + let dlobProvider : DLOBProvider; + if (useOrderSubscriber) { + let subscriptionConfig; + if (useWebsocket) { + subscriptionConfig = { + type: "websocket", + }; + } else { + subscriptionConfig = { + type: "polling", + frequency: ORDERBOOK_UPDATE_INTERVAL + }; + } + + const orderSubscriber = new OrderSubscriber({ + driftClient, + subscriptionConfig, + }); + + dlobProvider = getDLOBProviderFromOrderSubscriber(orderSubscriber); + + slotSource = { + getSlot: () => orderSubscriber.getSlot(), + }; + } else { + const userMap = new UserMap( + driftClient, + driftClient.userAccountSubscriptionConfig, + false + ); + + dlobProvider = getDLOBProviderFromUserMap(userMap); + } + const dlobCoder = DLOBOrdersCoder.create(); await driftClient.subscribe(); @@ -233,18 +272,13 @@ const main = async () => { const userStatsMap = new UserStatsMap(driftClient); - logger.info(`Initializing userMap...`); + logger.info(`Initializing DLOB Provider...`); const initUserMapStart = Date.now(); - const userMap = new UserMap( - driftClient, - driftClient.userAccountSubscriptionConfig, - false - ); - await userMap.subscribe(); - logger.info(`userMap initialized in ${Date.now() - initUserMapStart} ms`); + await dlobProvider.subscribe(); + logger.info(`dlob provider initialized in ${Date.now() - initUserMapStart} ms`); const initUserStatsMapStarts = Date.now(); - await userStatsMap.sync(userMap.getUniqueAuthorities()); + await userStatsMap.sync(dlobProvider.getUniqueAuthorities()); logger.info( `userStatsMap initialized in ${Date.now() - initUserStatsMapStarts} ms` ); @@ -253,7 +287,7 @@ const main = async () => { const initDlobSubscriberStart = Date.now(); const dlobSubscriber = new DLOBSubscriber({ driftClient, - dlobSource: userMap, + dlobSource: dlobProvider, slotSource, updateFrequency: ORDERBOOK_UPDATE_INTERVAL, }); @@ -267,7 +301,7 @@ const main = async () => { const handleStartup = async (_req, res, _next) => { if ( driftClient.isSubscribed && - userMap.size() > 0 && + dlobProvider.size() > 0 && userStatsMap.size() > 0 ) { res.writeHead(200); @@ -299,16 +333,14 @@ const main = async () => { }); } - for (const user of userMap.values()) { - const userAccount = user.getUserAccount(); - + for (const {userAccount, publicKey} of dlobProvider.getUserAccounts()) { for (const order of userAccount.orders) { if (isVariant(order.status, 'init')) { continue; } orders.push({ - user: user.getUserAccountPublicKey().toBase58(), + user: publicKey.toBase58(), order: order, }); } @@ -354,9 +386,7 @@ const main = async () => { } oracles.push(oracleHuman); } - for (const user of userMap.values()) { - const userAccount = user.getUserAccount(); - + for (const {userAccount, publicKey} of dlobProvider.getUserAccounts()) { for (const order of userAccount.orders) { if (isVariant(order.status, 'init')) { continue; @@ -394,7 +424,7 @@ const main = async () => { } orders.push({ - user: user.getUserAccountPublicKey().toBase58(), + user: publicKey.toBase58(), order: orderHuman, }); } @@ -418,16 +448,14 @@ const main = async () => { try { const dlobOrders: DLOBOrders = []; - for (const user of userMap.values()) { - const userAccount = user.getUserAccount(); - + for (const {userAccount, publicKey} of dlobProvider.getUserAccounts()) { for (const order of userAccount.orders) { if (isVariant(order.status, 'init')) { continue; } dlobOrders.push({ - user: user.getUserAccountPublicKey(), + user: publicKey, order, } as DLOBOrder); } @@ -468,9 +496,7 @@ const main = async () => { const dlobOrders: DLOBOrders = []; - for (const user of userMap.values()) { - const userAccount = user.getUserAccount(); - + for (const {userAccount, publicKey} of dlobProvider.getUserAccounts()) { for (const order of userAccount.orders) { if (isVariant(order.status, 'init')) { continue; @@ -486,7 +512,7 @@ const main = async () => { } dlobOrders.push({ - user: user.getUserAccountPublicKey(), + user: publicKey, order, } as DLOBOrder); } @@ -557,10 +583,9 @@ const main = async () => { continue; } else { if (`${includeUserStats}`.toLowerCase() === 'true') { - const userAccount = side.userAccount.toBase58(); - await userMap.mustGet(userAccount); + const userAccount = dlobProvider.getUserAccount(side.userAccount); const userStats = await userStatsMap.mustGet( - userMap.getUserAuthority(userAccount)!.toBase58() + userAccount.authority.toBase58() ); topMakers.add([ userAccount, From d1f2ad3538fcb710e283d07de26b230352fb1cb1 Mon Sep 17 00:00:00 2001 From: wphan Date: Tue, 28 Nov 2023 16:59:18 -0800 Subject: [PATCH 06/12] make health check more optimistic --- src/core/metrics.ts | 11 ++++++----- src/index.ts | 24 ++++++------------------ 2 files changed, 12 insertions(+), 23 deletions(-) diff --git a/src/core/metrics.ts b/src/core/metrics.ts index 344661a..99e9de0 100644 --- a/src/core/metrics.ts +++ b/src/core/metrics.ts @@ -134,11 +134,10 @@ const handleHealthCheck = async (req, res, next) => { if (lastHealthCheckState) { res.writeHead(200); res.end('OK'); - } else { - res.writeHead(500); - res.end(`NOK`); + lastHealthCheckPerformed = Date.now(); + return; } - return; + // always check if last check was unhealthy (give it another chance to recover) } const { lastSlotReceived, lastSlotReceivedMutex } = getSlotHealthCheckInfo(); @@ -149,7 +148,9 @@ const handleHealthCheck = async (req, res, next) => { lastHealthCheckState = lastSlotReceived > lastHealthCheckSlot; if (!lastHealthCheckState) { logger.error( - `Unhealthy: lastSlot: ${lastSlotReceived}, lastHealthCheckSlot: ${lastHealthCheckSlot}` + `Unhealthy: lastSlot: ${lastSlotReceived}, lastHealthCheckSlot: ${lastHealthCheckSlot}, timeSinceLastCheck: ${ + Date.now() - lastHealthCheckPerformed + } ms` ); } diff --git a/src/index.ts b/src/index.ts index 8caed26..7d06acb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -21,8 +21,8 @@ import { SlotSource, SlotSubscriber, UserMap, - UserStatsMap, Wallet, + getUserStatsAccountPublicKey, getVariant, groupL2, initialize, @@ -231,8 +231,6 @@ const main = async () => { lastSlotReceived = slotSource.getSlot(); }, ORDERBOOK_UPDATE_INTERVAL); - const userStatsMap = new UserStatsMap(driftClient); - logger.info(`Initializing userMap...`); const initUserMapStart = Date.now(); const userMap = new UserMap( @@ -243,12 +241,6 @@ const main = async () => { await userMap.subscribe(); logger.info(`userMap initialized in ${Date.now() - initUserMapStart} ms`); - const initUserStatsMapStarts = Date.now(); - await userStatsMap.sync(userMap.getUniqueAuthorities()); - logger.info( - `userStatsMap initialized in ${Date.now() - initUserStatsMapStarts} ms` - ); - logger.info(`Initializing DLOBSubscriber...`); const initDlobSubscriberStart = Date.now(); const dlobSubscriber = new DLOBSubscriber({ @@ -265,11 +257,7 @@ const main = async () => { MARKET_SUBSCRIBERS = await initializeAllMarketSubscribers(driftClient); const handleStartup = async (_req, res, _next) => { - if ( - driftClient.isSubscribed && - userMap.size() > 0 && - userStatsMap.size() > 0 - ) { + if (driftClient.isSubscribed && userMap.size() > 0) { res.writeHead(200); res.end('OK'); } else { @@ -559,12 +547,12 @@ const main = async () => { if (`${includeUserStats}`.toLowerCase() === 'true') { const userAccount = side.userAccount.toBase58(); await userMap.mustGet(userAccount); - const userStats = await userStatsMap.mustGet( - userMap.getUserAuthority(userAccount)!.toBase58() - ); topMakers.add([ userAccount, - userStats.userStatsAccountPublicKey.toBase58(), + getUserStatsAccountPublicKey( + driftClient.program.programId, + userMap.getUserAuthority(userAccount) + ), ]); } else { topMakers.add(side.userAccount.toBase58()); From ff83db8c2b6bd4088407f947ad57e9d7b4bd47d4 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 29 Nov 2023 01:27:27 +0000 Subject: [PATCH 07/12] Bumping sdk dependency to 2.48.0-beta.12 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 9b8a8f8..25bd411 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.48.0-beta.11", + "@drift-labs/sdk": "2.48.0-beta.12", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 09455f5..d8430ce 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.48.0-beta.11": - version "2.48.0-beta.11" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.48.0-beta.11.tgz#cd2f7b908754d49f62964ab894d3ab6c248dee3f" - integrity sha512-lhBlBUmKT3w9EMTDIBcEUdRgqZ6OHLfqPukxW4gVoa6eFqIpcaTk/DGTj3HCli6C684YwVHJfCQx4kXfffv6Sw== +"@drift-labs/sdk@2.48.0-beta.12": + version "2.48.0-beta.12" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.48.0-beta.12.tgz#255332eb4437468daa7851d6c918567a231318a6" + integrity sha512-rVKFsZjPTpelIjBbT6cxK0tCDb1M6s7oPba8v21RT+w1GIZeTCVNDBKvgbarIURhAjqgkg73TmY0hHZ+FsufNg== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 117927cb89b50cb842a489a073e1eef3c71dd0ba Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 29 Nov 2023 01:41:40 +0000 Subject: [PATCH 08/12] Bumping sdk dependency to 2.48.0-beta.13 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 25bd411..729e823 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.48.0-beta.12", + "@drift-labs/sdk": "2.48.0-beta.13", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index d8430ce..8f7a265 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.48.0-beta.12": - version "2.48.0-beta.12" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.48.0-beta.12.tgz#255332eb4437468daa7851d6c918567a231318a6" - integrity sha512-rVKFsZjPTpelIjBbT6cxK0tCDb1M6s7oPba8v21RT+w1GIZeTCVNDBKvgbarIURhAjqgkg73TmY0hHZ+FsufNg== +"@drift-labs/sdk@2.48.0-beta.13": + version "2.48.0-beta.13" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.48.0-beta.13.tgz#0ad1cfd4daac97e1859fe617a749a2676e462b63" + integrity sha512-ogtY38Db44PVZOr8WwNHijnwVMBLFvIYNxmlE9BPehdEgIQ9PYT9QXalanhQ1Ruc/Cs5DbMIT3XvjtLyfFzEUg== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From db4a6685425a34a2e8217514282d76a0580fe7ac Mon Sep 17 00:00:00 2001 From: Chris Heaney Date: Tue, 28 Nov 2023 21:14:45 -0500 Subject: [PATCH 09/12] updates to package.json and publisher --- package.json | 2 +- src/index.ts | 1 + src/publishers/dlobPublisher.ts | 48 ++++++++++++++++++++++++--------- yarn.lock | 8 +++--- 4 files changed, 42 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index 9b8a8f8..729e823 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.48.0-beta.11", + "@drift-labs/sdk": "2.48.0-beta.13", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/src/index.ts b/src/index.ts index f96be35..ad00bb5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -276,6 +276,7 @@ const main = async () => { const initUserMapStart = Date.now(); await dlobProvider.subscribe(); logger.info(`dlob provider initialized in ${Date.now() - initUserMapStart} ms`); + logger.info(`dlob provider size ${dlobProvider.size()}`); const initUserStatsMapStarts = Date.now(); await userStatsMap.sync(dlobProvider.getUniqueAuthorities()); diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index a598f30..a9d2a3e 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -9,13 +9,14 @@ import { SlotSubscriber, UserMap, Wallet, - BulkAccountLoader, + BulkAccountLoader, OrderSubscriber, SlotSource, } from '@drift-labs/sdk'; import { logger, setLogLevel } from '../utils/logger'; import { sleep } from '../utils/utils'; import { DLOBSubscriberIO } from '../dlob-subscriber/DLOBSubscriberIO'; import { RedisClient } from '../utils/redisClient'; +import {DLOBProvider, getDLOBProviderFromOrderSubscriber, getDLOBProviderFromUserMap} from "../dlobProvider"; require('dotenv').config(); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; @@ -37,6 +38,8 @@ setLogLevel(opts.debug ? 'debug' : 'info'); const endpoint = process.env.ENDPOINT; const wsEndpoint = process.env.WS_ENDPOINT; +const useOrderSubscriber = process.env.USE_ORDER_SUBSCRIBER?.toLowerCase() === 'true'; + logger.info(`RPC endpoint: ${endpoint}`); logger.info(`WS endpoint: ${wsEndpoint}`); logger.info(`DriftEnv: ${driftEnv}`); @@ -68,8 +71,6 @@ const main = async () => { env: driftEnv, }); - const slotSubscriber = new SlotSubscriber(connection, {}); - const lamportsBalance = await connection.getBalance(wallet.publicKey); logger.info( `DriftClient ProgramId: ${driftClient.program.programId.toBase58()}` @@ -83,22 +84,45 @@ const main = async () => { logger.error(e); }); - await slotSubscriber.subscribe(); + let slotSource: SlotSource; + let dlobProvider : DLOBProvider; + if (useOrderSubscriber) { + const orderSubscriber = new OrderSubscriber({ + driftClient, + subscriptionConfig: { + type: "polling", + frequency: ORDERBOOK_UPDATE_INTERVAL + }, + }); - const userMap = new UserMap( - driftClient, - driftClient.userAccountSubscriptionConfig, - false - ); - await userMap.subscribe(); + dlobProvider = getDLOBProviderFromOrderSubscriber(orderSubscriber); + + slotSource = { + getSlot: () => orderSubscriber.getSlot(), + }; + } else { + const userMap = new UserMap( + driftClient, + driftClient.userAccountSubscriptionConfig, + false + ); + + dlobProvider = getDLOBProviderFromUserMap(userMap); + + slotSource = { + getSlot: () => bulkAccountLoader.getSlot(), + }; + } + + await dlobProvider.subscribe(); const redisClient = new RedisClient(REDIS_HOST, REDIS_PORT, REDIS_PASSWORD); await redisClient.connect(); const dlobSubscriber = new DLOBSubscriberIO({ driftClient, - dlobSource: userMap, - slotSource: slotSubscriber, + dlobSource: dlobProvider, + slotSource, updateFrequency: ORDERBOOK_UPDATE_INTERVAL, redisClient, }); diff --git a/yarn.lock b/yarn.lock index 09455f5..8f7a265 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.48.0-beta.11": - version "2.48.0-beta.11" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.48.0-beta.11.tgz#cd2f7b908754d49f62964ab894d3ab6c248dee3f" - integrity sha512-lhBlBUmKT3w9EMTDIBcEUdRgqZ6OHLfqPukxW4gVoa6eFqIpcaTk/DGTj3HCli6C684YwVHJfCQx4kXfffv6Sw== +"@drift-labs/sdk@2.48.0-beta.13": + version "2.48.0-beta.13" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.48.0-beta.13.tgz#0ad1cfd4daac97e1859fe617a749a2676e462b63" + integrity sha512-ogtY38Db44PVZOr8WwNHijnwVMBLFvIYNxmlE9BPehdEgIQ9PYT9QXalanhQ1Ruc/Cs5DbMIT3XvjtLyfFzEUg== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From d66eb874df71685800584f2522dc89ed29a673b5 Mon Sep 17 00:00:00 2001 From: Chris Heaney Date: Tue, 28 Nov 2023 21:15:08 -0500 Subject: [PATCH 10/12] prettify:fix --- src/dlobProvider.ts | 42 +++++++++++++++++++++------------ src/index.ts | 33 ++++++++++++++++---------- src/publishers/dlobPublisher.ts | 19 ++++++++++----- 3 files changed, 61 insertions(+), 33 deletions(-) diff --git a/src/dlobProvider.ts b/src/dlobProvider.ts index bfa58f2..1470543 100644 --- a/src/dlobProvider.ts +++ b/src/dlobProvider.ts @@ -1,16 +1,19 @@ -import {DLOB, OrderSubscriber, UserAccount, UserMap} from "@drift-labs/sdk"; -import {PublicKey} from "@solana/web3.js"; +import { DLOB, OrderSubscriber, UserAccount, UserMap } from '@drift-labs/sdk'; +import { PublicKey } from '@solana/web3.js'; export type DLOBProvider = { - subscribe() : Promise; + subscribe(): Promise; getDLOB(slot: number): Promise; getUniqueAuthorities(): PublicKey[]; - getUserAccounts(): Generator<{userAccount: UserAccount, publicKey: PublicKey}>; + getUserAccounts(): Generator<{ + userAccount: UserAccount; + publicKey: PublicKey; + }>; getUserAccount(publicKey: PublicKey): UserAccount | undefined; size(): number; -} +}; -export function getDLOBProviderFromUserMap(userMap:UserMap) : DLOBProvider { +export function getDLOBProviderFromUserMap(userMap: UserMap): DLOBProvider { return { subscribe: async () => { await userMap.subscribe(); @@ -21,9 +24,12 @@ export function getDLOBProviderFromUserMap(userMap:UserMap) : DLOBProvider { getUniqueAuthorities: () => { return userMap.getUniqueAuthorities(); }, - getUserAccounts: function*() { + getUserAccounts: function* () { for (const user of userMap.values()) { - yield {userAccount: user.getUserAccount(), publicKey: user.getUserAccountPublicKey()}; + yield { + userAccount: user.getUserAccount(), + publicKey: user.getUserAccountPublicKey(), + }; } }, getUserAccount: (publicKey) => { @@ -31,11 +37,13 @@ export function getDLOBProviderFromUserMap(userMap:UserMap) : DLOBProvider { }, size: () => { return userMap.size(); - } + }, }; } -export function getDLOBProviderFromOrderSubscriber(orderSubscriber: OrderSubscriber) : DLOBProvider { +export function getDLOBProviderFromOrderSubscriber( + orderSubscriber: OrderSubscriber +): DLOBProvider { return { subscribe: async () => { await orderSubscriber.subscribe(); @@ -50,16 +58,20 @@ export function getDLOBProviderFromOrderSubscriber(orderSubscriber: OrderSubscri } return Array.from(authorities.values()); }, - getUserAccounts: function*() { - for (const [key, {userAccount}] of orderSubscriber.usersAccounts.entries()) { - yield {userAccount: userAccount, publicKey: new PublicKey(key)}; + getUserAccounts: function* () { + for (const [ + key, + { userAccount }, + ] of orderSubscriber.usersAccounts.entries()) { + yield { userAccount: userAccount, publicKey: new PublicKey(key) }; } }, getUserAccount: (publicKey) => { - return orderSubscriber.usersAccounts.get(publicKey.toString())?.userAccount; + return orderSubscriber.usersAccounts.get(publicKey.toString()) + ?.userAccount; }, size(): number { return orderSubscriber.usersAccounts.size; - } + }, }; } diff --git a/src/index.ts b/src/index.ts index ad00bb5..c960e32 100644 --- a/src/index.ts +++ b/src/index.ts @@ -47,7 +47,11 @@ import { sleep, validateDlobQuery, } from './utils/utils'; -import {DLOBProvider, getDLOBProviderFromOrderSubscriber, getDLOBProviderFromUserMap} from "./dlobProvider"; +import { + DLOBProvider, + getDLOBProviderFromOrderSubscriber, + getDLOBProviderFromUserMap, +} from './dlobProvider'; require('dotenv').config(); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; @@ -59,7 +63,8 @@ const stateCommitment: Commitment = 'processed'; const serverPort = process.env.PORT || 6969; export const ORDERBOOK_UPDATE_INTERVAL = 1000; const useWebsocket = process.env.USE_WEBSOCKET?.toLowerCase() === 'true'; -const useOrderSubscriber = process.env.USE_ORDER_SUBSCRIBER?.toLowerCase() === 'true'; +const useOrderSubscriber = + process.env.USE_ORDER_SUBSCRIBER?.toLowerCase() === 'true'; const rateLimitCallsPerSecond = process.env.RATE_LIMIT_CALLS_PER_SECOND ? parseInt(process.env.RATE_LIMIT_CALLS_PER_SECOND) @@ -224,17 +229,17 @@ const main = async () => { env: driftEnv, }); - let dlobProvider : DLOBProvider; + let dlobProvider: DLOBProvider; if (useOrderSubscriber) { let subscriptionConfig; if (useWebsocket) { subscriptionConfig = { - type: "websocket", + type: 'websocket', }; } else { subscriptionConfig = { - type: "polling", - frequency: ORDERBOOK_UPDATE_INTERVAL + type: 'polling', + frequency: ORDERBOOK_UPDATE_INTERVAL, }; } @@ -275,7 +280,9 @@ const main = async () => { logger.info(`Initializing DLOB Provider...`); const initUserMapStart = Date.now(); await dlobProvider.subscribe(); - logger.info(`dlob provider initialized in ${Date.now() - initUserMapStart} ms`); + logger.info( + `dlob provider initialized in ${Date.now() - initUserMapStart} ms` + ); logger.info(`dlob provider size ${dlobProvider.size()}`); const initUserStatsMapStarts = Date.now(); @@ -334,7 +341,7 @@ const main = async () => { }); } - for (const {userAccount, publicKey} of dlobProvider.getUserAccounts()) { + for (const { userAccount, publicKey } of dlobProvider.getUserAccounts()) { for (const order of userAccount.orders) { if (isVariant(order.status, 'init')) { continue; @@ -387,7 +394,7 @@ const main = async () => { } oracles.push(oracleHuman); } - for (const {userAccount, publicKey} of dlobProvider.getUserAccounts()) { + for (const { userAccount, publicKey } of dlobProvider.getUserAccounts()) { for (const order of userAccount.orders) { if (isVariant(order.status, 'init')) { continue; @@ -449,7 +456,7 @@ const main = async () => { try { const dlobOrders: DLOBOrders = []; - for (const {userAccount, publicKey} of dlobProvider.getUserAccounts()) { + for (const { userAccount, publicKey } of dlobProvider.getUserAccounts()) { for (const order of userAccount.orders) { if (isVariant(order.status, 'init')) { continue; @@ -497,7 +504,7 @@ const main = async () => { const dlobOrders: DLOBOrders = []; - for (const {userAccount, publicKey} of dlobProvider.getUserAccounts()) { + for (const { userAccount, publicKey } of dlobProvider.getUserAccounts()) { for (const order of userAccount.orders) { if (isVariant(order.status, 'init')) { continue; @@ -584,7 +591,9 @@ const main = async () => { continue; } else { if (`${includeUserStats}`.toLowerCase() === 'true') { - const userAccount = dlobProvider.getUserAccount(side.userAccount); + const userAccount = dlobProvider.getUserAccount( + side.userAccount + ); const userStats = await userStatsMap.mustGet( userAccount.authority.toBase58() ); diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index a9d2a3e..3ba8e6c 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -9,14 +9,20 @@ import { SlotSubscriber, UserMap, Wallet, - BulkAccountLoader, OrderSubscriber, SlotSource, + BulkAccountLoader, + OrderSubscriber, + SlotSource, } from '@drift-labs/sdk'; import { logger, setLogLevel } from '../utils/logger'; import { sleep } from '../utils/utils'; import { DLOBSubscriberIO } from '../dlob-subscriber/DLOBSubscriberIO'; import { RedisClient } from '../utils/redisClient'; -import {DLOBProvider, getDLOBProviderFromOrderSubscriber, getDLOBProviderFromUserMap} from "../dlobProvider"; +import { + DLOBProvider, + getDLOBProviderFromOrderSubscriber, + getDLOBProviderFromUserMap, +} from '../dlobProvider'; require('dotenv').config(); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; @@ -38,7 +44,8 @@ setLogLevel(opts.debug ? 'debug' : 'info'); const endpoint = process.env.ENDPOINT; const wsEndpoint = process.env.WS_ENDPOINT; -const useOrderSubscriber = process.env.USE_ORDER_SUBSCRIBER?.toLowerCase() === 'true'; +const useOrderSubscriber = + process.env.USE_ORDER_SUBSCRIBER?.toLowerCase() === 'true'; logger.info(`RPC endpoint: ${endpoint}`); logger.info(`WS endpoint: ${wsEndpoint}`); @@ -85,13 +92,13 @@ const main = async () => { }); let slotSource: SlotSource; - let dlobProvider : DLOBProvider; + let dlobProvider: DLOBProvider; if (useOrderSubscriber) { const orderSubscriber = new OrderSubscriber({ driftClient, subscriptionConfig: { - type: "polling", - frequency: ORDERBOOK_UPDATE_INTERVAL + type: 'polling', + frequency: ORDERBOOK_UPDATE_INTERVAL, }, }); From e5c9f0e96872a8b9874857027f9a5ee34d766be5 Mon Sep 17 00:00:00 2001 From: Chris Heaney Date: Tue, 28 Nov 2023 21:25:10 -0500 Subject: [PATCH 11/12] rm unused import --- src/publishers/dlobPublisher.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 3ba8e6c..9ad03d7 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -6,7 +6,6 @@ import { DriftClient, initialize, DriftEnv, - SlotSubscriber, UserMap, Wallet, BulkAccountLoader, From 506dc615383a183ce523691e4f4ac23f73374e51 Mon Sep 17 00:00:00 2001 From: Chris Heaney Date: Tue, 28 Nov 2023 21:27:52 -0500 Subject: [PATCH 12/12] tweak namings --- src/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index fd138a3..888b6a1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -276,10 +276,10 @@ const main = async () => { }, ORDERBOOK_UPDATE_INTERVAL); logger.info(`Initializing DLOB Provider...`); - const initUserMapStart = Date.now(); + const initDLOBProviderStart = Date.now(); await dlobProvider.subscribe(); logger.info( - `dlob provider initialized in ${Date.now() - initUserMapStart} ms` + `dlob provider initialized in ${Date.now() - initDLOBProviderStart} ms` ); logger.info(`dlob provider size ${dlobProvider.size()}`);