From 413f6f1898d0db1a2e9d1c64444abd9596b225e0 Mon Sep 17 00:00:00 2001 From: Luke Steyn Date: Tue, 30 May 2023 22:25:35 +0900 Subject: [PATCH 1/4] Drift for changes to DlobSubscriber --- src/index.ts | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/src/index.ts b/src/index.ts index cebcf32..a32ca71 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,7 +24,6 @@ import { PerpMarkets, DLOBSubscriber, MarketType, - isVariant, } from "@drift-labs/sdk"; import { Mutex } from "async-mutex"; @@ -686,8 +685,15 @@ const main = async () => { app.get("/l2", handleResponseTime, async (req, res, next) => { try { - const { marketName, marketIndex, marketType, depth, includeVamm } = - req.query; + const { + marketName, + marketIndex, + marketType, + depth, + includeVamm, + includePhoenix, + includeSerum, + } = req.query; const { normedMarketType, normedMarketIndex, error } = validateDlobQuery( marketType as string, @@ -699,13 +705,21 @@ const main = async () => { return; } - const l2 = dlobSubscriber.getL2({ + const l2 = await dlobSubscriber.getL2({ marketIndex: normedMarketIndex, marketType: normedMarketType, depth: depth ? parseInt(depth as string) : 10, - includeVamm: includeVamm - ? (includeVamm as string).toLowerCase() === "true" - : false, + opts: { + includeVammL2: includeVamm + ? `${includeVamm}`.toLowerCase() === "true" + : false, + includePhoenixL2: includeSerum + ? `${includeSerum}`.toLowerCase() === "true" + : false, + includeSerumL2: includePhoenix + ? `${includePhoenix}`.toLowerCase() === "true" + : false, + }, }); for (const key of Object.keys(l2)) { From 92f78dd540afe4c5583460ebd040b538ceaa6a2a Mon Sep 17 00:00:00 2001 From: Luke Steyn Date: Wed, 31 May 2023 16:02:17 +0900 Subject: [PATCH 2/4] Added phoenix and serum subscribers --- .vscode/launch.json | 16 +++++++ package.json | 101 ++++++++++++++++++++++---------------------- src/index.ts | 89 +++++++++++++++++++++++++++++++++----- 3 files changed, 145 insertions(+), 61 deletions(-) create mode 100644 .vscode/launch.json diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..fc4635d --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "type": "node", + "request": "attach", + "name": "Debug", + "port": 2230, + "timeout": 3000, + "restart": true, + }, + ] +} \ No newline at end of file diff --git a/package.json b/package.json index 5575ce0..d1ceec9 100644 --- a/package.json +++ b/package.json @@ -1,52 +1,53 @@ { - "name": "@drift-labs/dlob-server", - "version": "0.1.0", - "author": "wphan", - "main": "lib/index.js", - "license": "Apache-2.0", - "dependencies": { - "@drift-labs/sdk": "2.30.0", - "@opentelemetry/api": "^1.1.0", - "@opentelemetry/auto-instrumentations-node": "^0.31.1", - "@opentelemetry/exporter-prometheus": "^0.31.0", - "@opentelemetry/sdk-node": "^0.31.0", - "@project-serum/anchor": "^0.19.1-beta.1", - "@project-serum/serum": "^0.13.65", - "@solana/web3.js": "^1.22.0", - "async-mutex": "^0.4.0", - "commander": "^9.4.0", - "compression": "^1.7.4", - "cors": "^2.8.5", - "dotenv": "^10.0.0", - "express": "^4.18.2", - "express-rate-limit": "^6.7.0", - "morgan": "^1.10.0", - "response-time": "^2.3.2", - "typescript": "4.5.4", - "winston": "^3.8.1" - }, - "devDependencies": { - "@typescript-eslint/eslint-plugin": "^4.28.0", - "@typescript-eslint/parser": "^4.28.0", - "eslint": "^7.29.0", - "eslint-config-prettier": "^8.3.0", - "eslint-plugin-prettier": "^3.4.0", - "husky": "^7.0.4", - "prettier": "^2.4.1", - "ts-node": "^10.9.1" - }, - "scripts": { - "prepare": "husky install", - "build": "yarn clean && tsc", - "clean": "rm -rf lib", - "start": "node lib/index.js", - "dev": "ts-node src/index.ts", - "dev:inspect": "yarn build && node --inspect ./lib/index.js", - "example": "ts-node example/client.ts", - "exampleWithSlot": "ts-node example/clientWithSlot.ts", - "prettify": "prettier --check './src/**/*.ts'", - "prettify:fix": "prettier --write './src/**/*.ts'", - "lint": "eslint . --ext ts --quiet", - "lint:fix": "eslint . --ext ts --fix" - } + "name": "@drift-labs/dlob-server", + "version": "0.1.0", + "author": "wphan", + "main": "lib/index.js", + "license": "Apache-2.0", + "dependencies": { + "@drift-labs/sdk": "2.30.0", + "@opentelemetry/api": "^1.1.0", + "@opentelemetry/auto-instrumentations-node": "^0.31.1", + "@opentelemetry/exporter-prometheus": "^0.31.0", + "@opentelemetry/sdk-node": "^0.31.0", + "@project-serum/anchor": "^0.19.1-beta.1", + "@project-serum/serum": "^0.13.65", + "@solana/web3.js": "^1.22.0", + "async-mutex": "^0.4.0", + "commander": "^9.4.0", + "compression": "^1.7.4", + "cors": "^2.8.5", + "dotenv": "^10.0.0", + "express": "^4.18.2", + "express-rate-limit": "^6.7.0", + "morgan": "^1.10.0", + "response-time": "^2.3.2", + "typescript": "4.5.4", + "winston": "^3.8.1" + }, + "devDependencies": { + "@typescript-eslint/eslint-plugin": "^4.28.0", + "@typescript-eslint/parser": "^4.28.0", + "eslint": "^7.29.0", + "eslint-config-prettier": "^8.3.0", + "eslint-plugin-prettier": "^3.4.0", + "husky": "^7.0.4", + "prettier": "^2.4.1", + "ts-node": "^10.9.1" + }, + "scripts": { + "prepare": "husky install", + "build": "yarn clean && tsc", + "clean": "rm -rf lib", + "start": "node lib/index.js", + "dev": "ts-node src/index.ts", + "dev:inspect": "yarn build && node --inspect ./lib/index.js", + "dev:debug": "yarn build && node --inspect-brk --inspect=2230 ./lib/index.js", + "example": "ts-node example/client.ts", + "exampleWithSlot": "ts-node example/clientWithSlot.ts", + "prettify": "prettier --check './src/**/*.ts'", + "prettify:fix": "prettier --write './src/**/*.ts'", + "lint": "eslint . --ext ts --quiet", + "lint:fix": "eslint . --ext ts --fix" + } } diff --git a/src/index.ts b/src/index.ts index a32ca71..6911896 100644 --- a/src/index.ts +++ b/src/index.ts @@ -24,6 +24,9 @@ import { PerpMarkets, DLOBSubscriber, MarketType, + SpotMarketConfig, + PhoenixSubscriber, + SerumSubscriber, } from "@drift-labs/sdk"; import { Mutex } from "async-mutex"; @@ -215,6 +218,68 @@ const endpointResponseTimeHistogram = meter.createHistogram( } ); +const getPhoenixSubscriber = ( + driftClient: DriftClient, + marketConfig: SpotMarketConfig +) => { + return new PhoenixSubscriber({ + connection: driftClient.connection, + programId: new PublicKey(sdkConfig.PHOENIX), + marketAddress: marketConfig.phoenixMarket, + accountSubscription: { + type: "websocket", + }, + }); +}; + +const getSerumSubscriber = ( + driftClient: DriftClient, + marketConfig: SpotMarketConfig +) => { + return new SerumSubscriber({ + connection: driftClient.connection, + programId: new PublicKey(sdkConfig.SERUM_V3), + marketAddress: marketConfig.serumMarket, + accountSubscription: { + type: "websocket", + }, + }); +}; + +type SubscriberLookup = { + [marketIndex: number]: { + phoenix?: PhoenixSubscriber; + serum?: SerumSubscriber; + }; +}; + +let MARKET_SUBSCRIBERS: SubscriberLookup = {}; + +const initializeAllMarketSubscribers = async (driftClient: DriftClient) => { + const markets: SubscriberLookup = {}; + + for (const market of sdkConfig.SPOT_MARKETS) { + markets[market.marketIndex] = { + phoenix: undefined, + serum: undefined, + }; + + if (market.phoenixMarket) { + const phoenixSubscriber = getPhoenixSubscriber(driftClient, market); + await phoenixSubscriber.subscribe(); + markets[market.marketIndex].phoenix = phoenixSubscriber; + } + + if (market.serumMarket) { + const serumSubscriber = getSerumSubscriber(driftClient, market); + await serumSubscriber.subscribe(); + markets[market.marketIndex].serum = serumSubscriber; + } + } + + return markets; +}; + const main = async () => { const wallet = getWallet(); const clearingHousePublicKey = new PublicKey(sdkConfig.DRIFT_PROGRAM_ID); @@ -312,6 +377,8 @@ const main = async () => { }); }); + MARKET_SUBSCRIBERS = await initializeAllMarketSubscribers(driftClient); + // start http server listening to /health endpoint using http package app.get("/health", handleResponseTime, async (req, res, next) => { try { @@ -705,21 +772,21 @@ const main = async () => { return; } + const isSpot = marketType === "spot"; + const l2 = await dlobSubscriber.getL2({ marketIndex: normedMarketIndex, marketType: normedMarketType, depth: depth ? parseInt(depth as string) : 10, - opts: { - includeVammL2: includeVamm - ? `${includeVamm}`.toLowerCase() === "true" - : false, - includePhoenixL2: includeSerum - ? `${includeSerum}`.toLowerCase() === "true" - : false, - includeSerumL2: includePhoenix - ? `${includePhoenix}`.toLowerCase() === "true" - : false, - }, + includeVamm: `${includeVamm}`.toLowerCase() === "true", + fallbackL2Generators: isSpot + ? [ + `${includePhoenix}`.toLowerCase() === "true" && + MARKET_SUBSCRIBERS[normedMarketIndex].phoenix, + `${includeSerum}`.toLowerCase() === "true" && + MARKET_SUBSCRIBERS[normedMarketIndex].serum, + ].filter((a) => !!a) + : [], }); for (const key of Object.keys(l2)) { From a86a1f0e3ee4ca8b6385c21b160c2690ad427255 Mon Sep 17 00:00:00 2001 From: Luke Steyn Date: Wed, 31 May 2023 23:05:57 +0900 Subject: [PATCH 3/4] Updated subscribers to use polling --- src/index.ts | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/index.ts b/src/index.ts index 6911896..e5465b8 100644 --- a/src/index.ts +++ b/src/index.ts @@ -220,28 +220,32 @@ const endpointResponseTimeHistogram = meter.createHistogram( const getPhoenixSubscriber = ( driftClient: DriftClient, - marketConfig: SpotMarketConfig + marketConfig: SpotMarketConfig, + accountLoader: BulkAccountLoader ) => { return new PhoenixSubscriber({ connection: driftClient.connection, programId: new PublicKey(sdkConfig.PHOENIX), marketAddress: marketConfig.phoenixMarket, accountSubscription: { - type: "websocket", + type: "polling", + accountLoader, }, }); }; const getSerumSubscriber = ( driftClient: DriftClient, - marketConfig: SpotMarketConfig + marketConfig: SpotMarketConfig, + accountLoader: BulkAccountLoader ) => { return new SerumSubscriber({ connection: driftClient.connection, programId: new PublicKey(sdkConfig.SERUM_V3), marketAddress: marketConfig.serumMarket, accountSubscription: { - type: "websocket", + type: "polling", + accountLoader, }, }); }; @@ -255,7 +259,10 @@ type SubscriberLookup = { let MARKET_SUBSCRIBERS: SubscriberLookup = {}; -const initializeAllMarketSubscribers = async (driftClient: DriftClient) => { +const initializeAllMarketSubscribers = async ( + driftClient: DriftClient, + bulkAccountLoader: BulkAccountLoader +) => { const markets: SubscriberLookup = {}; for (const market of sdkConfig.SPOT_MARKETS) { @@ -265,13 +272,21 @@ const initializeAllMarketSubscribers = async (driftClient: DriftClient) => { }; if (market.phoenixMarket) { - const phoenixSubscriber = getPhoenixSubscriber(driftClient, market); + const phoenixSubscriber = getPhoenixSubscriber( + driftClient, + market, + bulkAccountLoader + ); await phoenixSubscriber.subscribe(); markets[market.marketIndex].phoenix = phoenixSubscriber; } if (market.serumMarket) { - const serumSubscriber = getSerumSubscriber(driftClient, market); + const serumSubscriber = getSerumSubscriber( + driftClient, + market, + bulkAccountLoader + ); await serumSubscriber.subscribe(); markets[market.marketIndex].serum = serumSubscriber; } @@ -377,7 +392,10 @@ const main = async () => { }); }); - MARKET_SUBSCRIBERS = await initializeAllMarketSubscribers(driftClient); + MARKET_SUBSCRIBERS = await initializeAllMarketSubscribers( + driftClient, + bulkAccountLoader + ); // start http server listening to /health endpoint using http package app.get("/health", handleResponseTime, async (req, res, next) => { From f22dbc05a74564417ee5bc1e452b11ea7df67fad Mon Sep 17 00:00:00 2001 From: Luke Steyn Date: Thu, 1 Jun 2023 00:39:24 +0900 Subject: [PATCH 4/4] PR feedback --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index e5465b8..7293ce7 100644 --- a/src/index.ts +++ b/src/index.ts @@ -790,7 +790,7 @@ const main = async () => { return; } - const isSpot = marketType === "spot"; + const isSpot = getVariant(normedMarketType); const l2 = await dlobSubscriber.getL2({ marketIndex: normedMarketIndex,