From 22203fd20a7e4501c7ebc1fdd6a9cd5571610a3f Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 13:25:32 +0000 Subject: [PATCH 01/38] Bumping sdk dependency to 2.42.0-beta.15 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index cde8518..d08ac1b 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.42.0-beta.14", + "@drift-labs/sdk": "2.42.0-beta.15", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 38e158f..937059e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.42.0-beta.14": - version "2.42.0-beta.14" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.42.0-beta.14.tgz#b29976ca4bcfdeab74191369f6197faece8d4b90" - integrity sha512-C3J3VDoXkWrk8S6z1zO4NNm7kZoqhAKMROD6WM4Yn4dKY/jDq6KwfjlVnZ7iwxsO1YTqoLP/N9N29JsLfHVqgg== +"@drift-labs/sdk@2.42.0-beta.15": + version "2.42.0-beta.15" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.42.0-beta.15.tgz#84208a34d09964bdf2e69a82bad61aa514efbe32" + integrity sha512-Gt6xxmGDKDJr+azs1PCdaa89urR+92FqTmAfjjmaWchAVzB4tma7iT9bo/5x7zlIuhzeILj9BvOY6tdl0l+GmQ== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From b012b4ade9c08087890863e71b693801de595711 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 24 Oct 2023 21:33:01 +0000 Subject: [PATCH 02/38] Bumping sdk dependency to 2.42.0-beta.16 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index d08ac1b..ccf34ae 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.42.0-beta.15", + "@drift-labs/sdk": "2.42.0-beta.16", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 937059e..9dc03d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.42.0-beta.15": - version "2.42.0-beta.15" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.42.0-beta.15.tgz#84208a34d09964bdf2e69a82bad61aa514efbe32" - integrity sha512-Gt6xxmGDKDJr+azs1PCdaa89urR+92FqTmAfjjmaWchAVzB4tma7iT9bo/5x7zlIuhzeILj9BvOY6tdl0l+GmQ== +"@drift-labs/sdk@2.42.0-beta.16": + version "2.42.0-beta.16" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.42.0-beta.16.tgz#cebc82306139c573c138742afd293c5466d5bdc3" + integrity sha512-kzhjiZF5Pas6I1pPVltjxz+H6b2Hi/7pCUscKrgl3Ot6mQ+UG2yorDJyWPIA89K18qPYKZ8+B5xC846bqgZUmQ== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 568ab5a239ce23854975cde03af747240be03499 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Tue, 24 Oct 2023 15:20:35 -0700 Subject: [PATCH 03/38] ws server works --- .gitignore | 1 + package.json | 5 +- src/dlob-subscriber/DLOBSubscriberIO.ts | 56 +++++++++++ src/index.ts | 84 +++++++++++----- src/utils.ts | 23 +++++ yarn.lock | 122 +++++++++++++++++++++++- 6 files changed, 260 insertions(+), 31 deletions(-) create mode 100644 src/dlob-subscriber/DLOBSubscriberIO.ts create mode 100644 src/utils.ts diff --git a/.gitignore b/.gitignore index e0e8c8c..61ce045 100644 --- a/.gitignore +++ b/.gitignore @@ -140,3 +140,4 @@ src/**.js.map .env lib +src/playground.ts diff --git a/package.json b/package.json index d08ac1b..03ba8a7 100644 --- a/package.json +++ b/package.json @@ -22,6 +22,8 @@ "express-rate-limit": "^6.7.0", "morgan": "^1.10.0", "response-time": "^2.3.2", + "socket.io": "^4.7.2", + "socket.io-client": "^4.7.2", "typescript": "4.5.4", "winston": "^3.8.1" }, @@ -50,6 +52,7 @@ "prettify": "prettier --check './src/**/*.ts'", "prettify:fix": "prettier --write './src/**/*.ts'", "lint": "eslint . --ext ts --quiet", - "lint:fix": "eslint . --ext ts --fix" + "lint:fix": "eslint . --ext ts --fix", + "playground": "ts-node src/playground.ts" } } diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts new file mode 100644 index 0000000..c87cbfa --- /dev/null +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -0,0 +1,56 @@ +import { + BN, + DLOBSubscriber, + DLOBSubscriptionConfig, + L2OrderBookGenerator, + MarketType, + groupL2, +} from '@drift-labs/sdk'; +import { l2WithBNToStrings } from '../utils'; +import { Server } from 'socket.io'; +import { getIOServer } from '..'; + +type wsMarketL2Args = { + marketIndex: number; + marketType: MarketType; + marketName: string; + depth: number; + includeVamm: boolean; + grouping?: number; + fallbackL2Generators?: L2OrderBookGenerator[]; +}; + +export class DLOBSubscriberIO extends DLOBSubscriber { + public marketL2Args: wsMarketL2Args[] = []; + io: Server; + + constructor(config: DLOBSubscriptionConfig) { + super(config); + } + + override async updateDLOB(): Promise { + this.io = getIOServer(); + await super.updateDLOB(); + for (const l2Args of this.marketL2Args) { + this.getL2AndSendMsg(l2Args); + } + } + + getL2AndSendMsg(l2Args: wsMarketL2Args): void { + console.time('getL2AndSendMsg'); + const grouping = l2Args.grouping; + const l2 = this.getL2(l2Args); + let l2Formatted: any; + if (grouping) { + const groupingBN = new BN(grouping); + l2Formatted = l2WithBNToStrings(groupL2(l2, groupingBN, l2Args.depth)); + } else { + l2Formatted = l2WithBNToStrings(l2); + } + l2Formatted['marketName'] = l2Args.marketName; + l2Formatted['marketType'] = l2Args.marketType; + l2Formatted['marketIndex'] = l2Args.marketIndex; + this.io.emit('l2Update', l2Formatted); + console.timeEnd('getL2AndSendMsg'); + } +} diff --git a/src/index.ts b/src/index.ts index 88df067..ef2d26f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -22,7 +22,6 @@ import { DLOBOrdersCoder, SpotMarkets, PerpMarkets, - DLOBSubscriber, MarketType, SpotMarketConfig, PhoenixSubscriber, @@ -31,7 +30,6 @@ import { isVariant, BN, groupL2, - L2OrderBook, Wallet, UserStatsMap, } from '@drift-labs/sdk'; @@ -48,6 +46,10 @@ import { View, } from '@opentelemetry/sdk-metrics-base'; import { ObservableResult } from '@opentelemetry/api'; +import * as http from 'http'; +import { Server } from 'socket.io'; +import { l2WithBNToStrings, sleep } from './utils'; +import { DLOBSubscriberIO } from './dlob-subscriber/DLOBSubscriberIO'; require('dotenv').config(); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; @@ -57,6 +59,12 @@ const sdkConfig = initialize({ env: process.env.ENV }); const stateCommitment: Commitment = 'processed'; const serverPort = process.env.PORT || 6969; +const maxPerpMarketIndex = Math.max( + ...sdkConfig.PERP_MARKETS.map((m) => m.marketIndex) +); +const maxSpotMarketIndex = Math.max( + ...sdkConfig.SPOT_MARKETS.map((m) => m.marketIndex) +); const bulkAccountLoaderPollingInterval = process.env .BULK_ACCOUNT_LOADER_POLLING_INTERVAL @@ -89,6 +97,13 @@ app.use(compression()); app.set('trust proxy', 1); app.use(logHttp); +const server = http.createServer(app); +const io = new Server(server); + +export function getIOServer(): Server { + return io; +} + const handleResponseTime = responseTime( (req: Request, res: Response, time: number) => { const endpoint = req.path; @@ -147,10 +162,6 @@ logger.info(`WS endpoint: ${wsEndpoint}`); logger.info(`DriftEnv: ${driftEnv}`); logger.info(`Commit: ${commitHash}`); -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - /** * Creates {count} buckets of size {increment} starting from {start}. Each bucket stores the count of values within its "size". * @param start @@ -398,7 +409,7 @@ const main = async () => { ); await userStatsMap.subscribe(); - const dlobSubscriber = new DLOBSubscriber({ + const dlobSubscriber = new DLOBSubscriberIO({ driftClient, dlobSource: userMap, slotSource: slotSubscriber, @@ -903,24 +914,6 @@ const main = async () => { } }); - const l2WithBNToStrings = (l2: L2OrderBook): any => { - for (const key of Object.keys(l2)) { - for (const idx in l2[key]) { - const level = l2[key][idx]; - const sources = level['sources']; - for (const sourceKey of Object.keys(sources)) { - sources[sourceKey] = sources[sourceKey].toString(); - } - l2[key][idx] = { - price: level.price.toString(), - size: level.size.toString(), - sources, - }; - } - } - return l2; - }; - const getOracleForMarket = ( marketType: MarketType, marketIndex: number @@ -1249,9 +1242,48 @@ const main = async () => { }); app.use(errorHandler); - app.listen(serverPort, () => { + server.listen(serverPort, () => { logger.info(`DLOB server listening on port http://localhost:${serverPort}`); }); + + /** + * Handle websocket logic here + */ + + io.on('connection', (socket) => { + console.log('a user connected'); + + socket.on('disconnect', () => { + console.log('user disconnected'); + }); + + socket.on('message', (msg: any) => { + const parsed = JSON.parse(msg); + if (parsed['type'] == 'subscribe') { + console.log('here'); + dlobSubscriber.marketL2Args.push({ + marketIndex: parseInt(parsed['marketIndex']), + marketType: isVariant(parsed['marketType'].toLowerCase(), 'spot') + ? MarketType.SPOT + : MarketType.PERP, + marketName: parsed['marketName'], + depth: parseInt(parsed['depth']), + includeVamm: parsed['includeVamm'] === 'true', + grouping: parsed['grouping'] + ? parseInt(parsed['grouping']) + : undefined, + fallbackL2Generators: isVariant(parsed['marketType'], 'spot') + ? [ + `${parsed['includePhoenix']}`.toLowerCase() === 'true' && + MARKET_SUBSCRIBERS[parseInt(parsed['marketIndex'])].phoenix, + `${parsed['includeSerum']}`.toLowerCase() === 'true' && + MARKET_SUBSCRIBERS[parseInt(parsed['marketIndex'])].serum, + ].filter((a) => !!a) + : [], + }); + } + }); + }); }; async function recursiveTryCatch(f: () => void) { diff --git a/src/utils.ts b/src/utils.ts new file mode 100644 index 0000000..73d8fa1 --- /dev/null +++ b/src/utils.ts @@ -0,0 +1,23 @@ +import { L2OrderBook } from '@drift-labs/sdk'; + +export const l2WithBNToStrings = (l2: L2OrderBook): any => { + for (const key of Object.keys(l2)) { + for (const idx in l2[key]) { + const level = l2[key][idx]; + const sources = level['sources']; + for (const sourceKey of Object.keys(sources)) { + sources[sourceKey] = sources[sourceKey].toString(); + } + l2[key][idx] = { + price: level.price.toString(), + size: level.size.toString(), + sources, + }; + } + } + return l2; +}; + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} diff --git a/yarn.lock b/yarn.lock index 937059e..2195452 100644 --- a/yarn.lock +++ b/yarn.lock @@ -901,6 +901,11 @@ resolved "https://registry.yarnpkg.com/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df" integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ== +"@socket.io/component-emitter@~3.1.0": + version "3.1.0" + resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553" + integrity sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg== + "@solana/buffer-layout-utils@^0.2.0": version "0.2.0" resolved "https://registry.yarnpkg.com/@solana/buffer-layout-utils/-/buffer-layout-utils-0.2.0.tgz#b45a6cab3293a2eb7597cceb474f229889d875ca" @@ -1090,6 +1095,11 @@ resolved "https://registry.yarnpkg.com/@types/content-disposition/-/content-disposition-0.5.5.tgz#650820e95de346e1f84e30667d168c8fd25aa6e3" integrity sha512-v6LCdKfK6BwcqMo+wYW05rLS12S0ZO0Fl4w1h4aaZMD7bqT3gVUns6FvLJKGZHQmYn3SX55JWGpziwJRwVgutA== +"@types/cookie@^0.4.1": + version "0.4.1" + resolved "https://registry.yarnpkg.com/@types/cookie/-/cookie-0.4.1.tgz#bfd02c1f2224567676c1545199f87c3a861d878d" + integrity sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q== + "@types/cookies@*": version "0.7.7" resolved "https://registry.yarnpkg.com/@types/cookies/-/cookies-0.7.7.tgz#7a92453d1d16389c05a5301eef566f34946cfd81" @@ -1100,6 +1110,13 @@ "@types/keygrip" "*" "@types/node" "*" +"@types/cors@^2.8.12": + version "2.8.15" + resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.15.tgz#eb143aa2f8807ddd78e83cbff141bbedd91b60ee" + integrity sha512-n91JxbNLD8eQIuXDIChAN1tCKNWCEgpceU9b7ZMbFA+P+Q4yIeh80jizFLEvolRPc1ES0VdwFlGv+kJTSirogw== + dependencies: + "@types/node" "*" + "@types/express-serve-static-core@^4.17.18": version "4.17.31" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.31.tgz#a1139efeab4e7323834bb0226e62ac019f474b2f" @@ -1280,6 +1297,13 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.9.tgz#02d013de7058cea16d36168ef2fc653464cfbad4" integrity sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg== +"@types/node@>=10.0.0": + version "20.8.8" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.8.tgz#adee050b422061ad5255fc38ff71b2bb96ea2a0e" + integrity sha512-YRsdVxq6OaLfmR9Hy816IMp33xOBjfyOgUd77ehqg96CFywxAPbDbXvAsuN2KVg2HOT8Eh6uAfU+l4WffwPVrQ== + dependencies: + undici-types "~5.25.1" + "@types/node@^12.12.54": version "12.20.55" resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.55.tgz#c329cbd434c42164f846b909bd6f85b5537f6240" @@ -1438,7 +1462,7 @@ abstract-logging@^2.0.0: resolved "https://registry.yarnpkg.com/abstract-logging/-/abstract-logging-2.0.1.tgz#6b0c371df212db7129b57d2e7fcf282b8bf1c839" integrity sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA== -accepts@~1.3.5, accepts@~1.3.8: +accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.8: version "1.3.8" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" integrity sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw== @@ -1620,6 +1644,11 @@ base64-js@^1.3.1, base64-js@^1.5.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== +base64id@2.0.0, base64id@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/base64id/-/base64id-2.0.0.tgz#2770ac6bc47d312af97a8bf9a634342e0cd25cb6" + integrity sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog== + basic-auth@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-2.0.1.tgz#b998279bf47ce38344b4f3cf916d4679bbf51e3a" @@ -1910,7 +1939,12 @@ cookie@0.5.0, cookie@^0.5.0: resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== -cors@^2.8.5: +cookie@~0.4.1: + version "0.4.2" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.2.tgz#0e41f24de5ecf317947c82fc789e06a884824432" + integrity sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA== + +cors@^2.8.5, cors@~2.8.5: version "2.8.5" resolved "https://registry.yarnpkg.com/cors/-/cors-2.8.5.tgz#eac11da51592dd86b9f06f6e7ac293b3df875d29" integrity sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g== @@ -1951,7 +1985,7 @@ debug@2.6.9: dependencies: ms "2.0.0" -debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.3, debug@^4.3.4: +debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.3, debug@^4.3.4, debug@~4.3.1, debug@~4.3.2: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== @@ -2065,6 +2099,38 @@ end-of-stream@^1.4.1: dependencies: once "^1.4.0" +engine.io-client@~6.5.2: + version "6.5.2" + resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-6.5.2.tgz#8709e22c291d4297ae80318d3c8baeae71f0e002" + integrity sha512-CQZqbrpEYnrpGqC07a9dJDz4gePZUgTPMU3NKJPSeQOyw27Tst4Pl3FemKoFGAlHzgZmKjoRmiJvbWfhCXUlIg== + dependencies: + "@socket.io/component-emitter" "~3.1.0" + debug "~4.3.1" + engine.io-parser "~5.2.1" + ws "~8.11.0" + xmlhttprequest-ssl "~2.0.0" + +engine.io-parser@~5.2.1: + version "5.2.1" + resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.2.1.tgz#9f213c77512ff1a6cc0c7a86108a7ffceb16fcfb" + integrity sha512-9JktcM3u18nU9N2Lz3bWeBgxVgOKpw7yhRaoxQA3FUDZzzw+9WlA6p4G4u0RixNkg14fH7EfEc/RhpurtiROTQ== + +engine.io@~6.5.2: + version "6.5.3" + resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-6.5.3.tgz#80b0692912cef3a417e1b7433301d6397bf0374b" + integrity sha512-IML/R4eG/pUS5w7OfcDE0jKrljWS9nwnEfsxWCIJF5eO6AHo6+Hlv+lQbdlAYsiJPHzUthLm1RUjnBzWOs45cw== + dependencies: + "@types/cookie" "^0.4.1" + "@types/cors" "^2.8.12" + "@types/node" ">=10.0.0" + accepts "~1.3.4" + base64id "2.0.0" + cookie "~0.4.1" + cors "~2.8.5" + debug "~4.3.1" + engine.io-parser "~5.2.1" + ws "~8.11.0" + enquirer@^2.3.5: version "2.3.6" resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" @@ -3575,6 +3641,44 @@ snake-case@^3.0.4: dot-case "^3.0.4" tslib "^2.0.3" +socket.io-adapter@~2.5.2: + version "2.5.2" + resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-2.5.2.tgz#5de9477c9182fdc171cd8c8364b9a8894ec75d12" + integrity sha512-87C3LO/NOMc+eMcpcxUBebGjkpMDkNBS9tf7KJqcDsmL936EChtVva71Dw2q4tQcuVC+hAUy4an2NO/sYXmwRA== + dependencies: + ws "~8.11.0" + +socket.io-client@^4.7.2: + version "4.7.2" + resolved "https://registry.yarnpkg.com/socket.io-client/-/socket.io-client-4.7.2.tgz#f2f13f68058bd4e40f94f2a1541f275157ff2c08" + integrity sha512-vtA0uD4ibrYD793SOIAwlo8cj6haOeMHrGvwPxJsxH7CeIksqJ+3Zc06RvWTIFgiSqx4A3sOnTXpfAEE2Zyz6w== + dependencies: + "@socket.io/component-emitter" "~3.1.0" + debug "~4.3.2" + engine.io-client "~6.5.2" + socket.io-parser "~4.2.4" + +socket.io-parser@~4.2.4: + version "4.2.4" + resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.4.tgz#c806966cf7270601e47469ddeec30fbdfda44c83" + integrity sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew== + dependencies: + "@socket.io/component-emitter" "~3.1.0" + debug "~4.3.1" + +socket.io@^4.7.2: + version "4.7.2" + resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.7.2.tgz#22557d76c3f3ca48f82e73d68b7add36a22df002" + integrity sha512-bvKVS29/I5fl2FGLNHuXlQaUH/BlzX1IN6S+NKLNZpBsPZIDH+90eQmCs2Railn4YUiww4SzUedJ6+uzwFnKLw== + dependencies: + accepts "~1.3.4" + base64id "~2.0.0" + cors "~2.8.5" + debug "~4.3.2" + engine.io "~6.5.2" + socket.io-adapter "~2.5.2" + socket.io-parser "~4.2.4" + sonic-boom@^1.0.2: version "1.4.1" resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-1.4.1.tgz#d35d6a74076624f12e6f917ade7b9d75e918f53e" @@ -3826,6 +3930,11 @@ typescript@4.5.4: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.4.tgz#a17d3a0263bf5c8723b9c52f43c5084edf13c2e8" integrity sha512-VgYs2A2QIRuGphtzFV7aQJduJ2gyfTljngLzjpfW9FoYZF6xuw1W0vW9ghCKLfcWrCFxK81CSGRAvS1pn4fIUg== +undici-types@~5.25.1: + version "5.25.3" + resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.25.3.tgz#e044115914c85f0bcbb229f346ab739f064998c3" + integrity sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA== + unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" @@ -3959,11 +4068,16 @@ ws@^7.4.5: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== -ws@^8.5.0: +ws@^8.5.0, ws@~8.11.0: version "8.11.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.11.0.tgz#6a0d36b8edfd9f96d8b25683db2f8d7de6e8e143" integrity sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg== +xmlhttprequest-ssl@~2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz#91360c86b914e67f44dce769180027c0da618c67" + integrity sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A== + xtend@^4.0.0: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" From 255ef1f7eb4be18c87640c5fe15816a2c0b580a8 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Tue, 24 Oct 2023 17:12:16 -0700 Subject: [PATCH 04/38] rename folder and actually sanity check --- src/core/metrics.ts | 142 ++++++ src/core/middleware.ts | 26 ++ src/dlob-subscriber/DLOBSubscriberIO.ts | 9 +- src/index.ts | 547 +++--------------------- src/utils.ts | 23 - src/{ => utils}/logger.ts | 0 src/utils/utils.ts | 282 ++++++++++++ 7 files changed, 513 insertions(+), 516 deletions(-) create mode 100644 src/core/metrics.ts create mode 100644 src/core/middleware.ts delete mode 100644 src/utils.ts rename src/{ => utils}/logger.ts (100%) create mode 100644 src/utils/utils.ts diff --git a/src/core/metrics.ts b/src/core/metrics.ts new file mode 100644 index 0000000..4eaabca --- /dev/null +++ b/src/core/metrics.ts @@ -0,0 +1,142 @@ +import { ObservableResult } from '@opentelemetry/api'; +import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; +import { logger } from '../utils/logger'; +import { + ExplicitBucketHistogramAggregation, + InstrumentType, + MeterProvider, + View, +} from '@opentelemetry/sdk-metrics-base'; +import { commitHash, driftEnv, endpoint, wsEndpoint } from '..'; + +/** + * Creates {count} buckets of size {increment} starting from {start}. Each bucket stores the count of values within its "size". + * @param start + * @param increment + * @param count + * @returns + */ +const createHistogramBuckets = ( + start: number, + increment: number, + count: number +) => { + return new ExplicitBucketHistogramAggregation( + Array.from(new Array(count), (_, i) => start + i * increment) + ); +}; + +enum METRIC_TYPES { + runtime_specs = 'runtime_specs', + endpoint_response_times_histogram = 'endpoint_response_times_histogram', + endpoint_response_status = 'endpoint_response_status', + health_status = 'health_status', +} + +export enum HEALTH_STATUS { + Ok = 0, + StaleBulkAccountLoader, + UnhealthySlotSubscriber, + LivenessTesting, +} + +const metricsPort = + parseInt(process.env.METRICS_PORT) || PrometheusExporter.DEFAULT_OPTIONS.port; +const { endpoint: defaultEndpoint } = PrometheusExporter.DEFAULT_OPTIONS; +const exporter = new PrometheusExporter( + { + port: metricsPort, + endpoint: defaultEndpoint, + }, + () => { + logger.info( + `prometheus scrape endpoint started: http://localhost:${metricsPort}${defaultEndpoint}` + ); + } +); +const meterName = 'dlob-meter'; +const meterProvider = new MeterProvider({ + views: [ + new View({ + instrumentName: METRIC_TYPES.endpoint_response_times_histogram, + instrumentType: InstrumentType.HISTOGRAM, + meterName, + aggregation: createHistogramBuckets(0, 20, 30), + }), + ], +}); +meterProvider.addMetricReader(exporter); +const meter = meterProvider.getMeter(meterName); + +const runtimeSpecsGauge = meter.createObservableGauge( + METRIC_TYPES.runtime_specs, + { + description: 'Runtime sepcification of this program', + } +); +const bootTimeMs = Date.now(); +runtimeSpecsGauge.addCallback((obs) => { + obs.observe(bootTimeMs, { + commit: commitHash, + driftEnv, + rpcEndpoint: endpoint, + wsEndpoint: wsEndpoint, + }); +}); + +let healthStatus: HEALTH_STATUS = HEALTH_STATUS.Ok; +const healthStatusGauge = meter.createObservableGauge( + METRIC_TYPES.health_status, + { + description: 'Health status of this program', + } +); +healthStatusGauge.addCallback((obs: ObservableResult) => { + obs.observe(healthStatus, {}); +}); + +const endpointResponseTimeHistogram = meter.createHistogram( + METRIC_TYPES.endpoint_response_times_histogram, + { + description: 'Duration of endpoint responses', + unit: 'ms', + } +); + +const responseStatusCounter = meter.createCounter( + METRIC_TYPES.endpoint_response_status, + { + description: 'Count of endpoint responses by status code', + } +); + +const startupTime = Date.now(); +const handleHealthCheck = async (req, res, next) => { + try { + if (req.url === '/health' || req.url === '/') { + if (Date.now() > startupTime + 60 * 1000) { + healthStatus = HEALTH_STATUS.LivenessTesting; + + res.writeHead(500); + res.end('Testing liveness test fail'); + return; + } + + // liveness check passed + healthStatus = HEALTH_STATUS.Ok; + res.writeHead(200); + res.end('OK'); + } else { + res.writeHead(404); + res.end('Not found'); + } + } catch (e) { + next(e); + } +}; + +export { + endpointResponseTimeHistogram, + responseStatusCounter, + handleHealthCheck, +}; diff --git a/src/core/middleware.ts b/src/core/middleware.ts new file mode 100644 index 0000000..7705d3c --- /dev/null +++ b/src/core/middleware.ts @@ -0,0 +1,26 @@ +import { Request, Response } from 'express'; +import responseTime = require('response-time'); +import { + endpointResponseTimeHistogram, + responseStatusCounter, +} from './metrics'; + +export const handleResponseTime = responseTime( + (req: Request, res: Response, time: number) => { + const endpoint = req.path; + + if (endpoint === '/health' || req.url === '/') { + return; + } + + responseStatusCounter.add(1, { + endpoint, + status: res.statusCode, + }); + + const responseTimeMs = time; + endpointResponseTimeHistogram.record(responseTimeMs, { + endpoint, + }); + } +); diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index c87cbfa..6f1b586 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -6,7 +6,7 @@ import { MarketType, groupL2, } from '@drift-labs/sdk'; -import { l2WithBNToStrings } from '../utils'; +import { getOracleForMarket, l2WithBNToStrings } from '../utils/utils'; import { Server } from 'socket.io'; import { getIOServer } from '..'; @@ -37,7 +37,6 @@ export class DLOBSubscriberIO extends DLOBSubscriber { } getL2AndSendMsg(l2Args: wsMarketL2Args): void { - console.time('getL2AndSendMsg'); const grouping = l2Args.grouping; const l2 = this.getL2(l2Args); let l2Formatted: any; @@ -50,7 +49,11 @@ export class DLOBSubscriberIO extends DLOBSubscriber { l2Formatted['marketName'] = l2Args.marketName; l2Formatted['marketType'] = l2Args.marketType; l2Formatted['marketIndex'] = l2Args.marketIndex; + l2Formatted['oracle'] = getOracleForMarket( + this.driftClient, + l2Args.marketType, + l2Args.marketIndex + ); this.io.emit('l2Update', l2Formatted); - console.timeEnd('getL2AndSendMsg'); } } diff --git a/src/index.ts b/src/index.ts index ef2d26f..f4343a2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,5 @@ import { program } from 'commander'; - -import responseTime = require('response-time'); -import express, { Request, Response } from 'express'; +import express from 'express'; import rateLimit from 'express-rate-limit'; import compression from 'compression'; import morgan from 'morgan'; @@ -11,7 +9,6 @@ import { Connection, Commitment, PublicKey, Keypair } from '@solana/web3.js'; import { getVariant, - BulkAccountLoader, DriftClient, initialize, DriftEnv, @@ -20,12 +17,7 @@ import { DLOBOrder, DLOBOrders, DLOBOrdersCoder, - SpotMarkets, - PerpMarkets, MarketType, - SpotMarketConfig, - PhoenixSubscriber, - SerumSubscriber, DLOBNode, isVariant, BN, @@ -34,22 +26,25 @@ import { UserStatsMap, } from '@drift-labs/sdk'; -import { Mutex } from 'async-mutex'; +import { logger, setLogLevel } from './utils/logger'; -import { logger, setLogLevel } from './logger'; - -import { PrometheusExporter } from '@opentelemetry/exporter-prometheus'; -import { - ExplicitBucketHistogramAggregation, - InstrumentType, - MeterProvider, - View, -} from '@opentelemetry/sdk-metrics-base'; -import { ObservableResult } from '@opentelemetry/api'; import * as http from 'http'; import { Server } from 'socket.io'; -import { l2WithBNToStrings, sleep } from './utils'; +import { + l2WithBNToStrings, + sleep, + getOracleForMarket, + normalizeBatchQueryParams, + SubscriberLookup, + errorHandler, + getPhoenixSubscriber, + getSerumSubscriber, + validateWsSubscribeMsg, + validateDlobQuery, +} from './utils/utils'; import { DLOBSubscriberIO } from './dlob-subscriber/DLOBSubscriberIO'; +import { handleResponseTime } from './core/middleware'; +import { handleHealthCheck } from './core/metrics'; require('dotenv').config(); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; @@ -59,23 +54,11 @@ const sdkConfig = initialize({ env: process.env.ENV }); const stateCommitment: Commitment = 'processed'; const serverPort = process.env.PORT || 6969; -const maxPerpMarketIndex = Math.max( - ...sdkConfig.PERP_MARKETS.map((m) => m.marketIndex) -); -const maxSpotMarketIndex = Math.max( - ...sdkConfig.SPOT_MARKETS.map((m) => m.marketIndex) -); - -const bulkAccountLoaderPollingInterval = process.env - .BULK_ACCOUNT_LOADER_POLLING_INTERVAL - ? parseInt(process.env.BULK_ACCOUNT_LOADER_POLLING_INTERVAL) - : 5000; -const healthCheckInterval = bulkAccountLoaderPollingInterval * 2; +const ORDERBOOK_UPDATE_INTERVAL = 100; const rateLimitCallsPerSecond = process.env.RATE_LIMIT_CALLS_PER_SECOND ? parseInt(process.env.RATE_LIMIT_CALLS_PER_SECOND) : 1; - const loadTestAllowed = process.env.ALLOW_LOAD_TEST?.toLowerCase() === 'true'; const logFormat = @@ -84,47 +67,14 @@ const logHttp = morgan(logFormat, { skip: (_req, res) => res.statusCode < 400, }); -function errorHandler(err, _req, res, _next) { - logger.error(`errorHandler, message: ${err.message}, stack: ${err.stack}`); - if (!res.headersSent) { - res.status(500).send('Internal error'); - } -} +let driftClient: DriftClient; const app = express(); app.use(cors({ origin: '*' })); app.use(compression()); app.set('trust proxy', 1); app.use(logHttp); - -const server = http.createServer(app); -const io = new Server(server); - -export function getIOServer(): Server { - return io; -} - -const handleResponseTime = responseTime( - (req: Request, res: Response, time: number) => { - const endpoint = req.path; - - if (endpoint === '/health' || req.url === '/') { - return; - } - - responseStatusCounter.add(1, { - endpoint, - status: res.statusCode, - }); - - const responseTimeMs = time; - endpointResponseTimeHistogram.record(responseTimeMs, { - endpoint, - }); - } -); app.use(handleResponseTime); - app.use( rateLimit({ windowMs: 1000, // 1 second @@ -152,6 +102,14 @@ app.use((req, _res, next) => { next(); }); +app.use(errorHandler); +const server = http.createServer(app); +const io = new Server(server); + +export function getIOServer(): Server { + return io; +} + const opts = program.opts(); setLogLevel(opts.debug ? 'debug' : 'info'); @@ -162,152 +120,9 @@ logger.info(`WS endpoint: ${wsEndpoint}`); logger.info(`DriftEnv: ${driftEnv}`); logger.info(`Commit: ${commitHash}`); -/** - * Creates {count} buckets of size {increment} starting from {start}. Each bucket stores the count of values within its "size". - * @param start - * @param increment - * @param count - * @returns - */ -const createHistogramBuckets = ( - start: number, - increment: number, - count: number -) => { - return new ExplicitBucketHistogramAggregation( - Array.from(new Array(count), (_, i) => start + i * increment) - ); -}; - -enum METRIC_TYPES { - runtime_specs = 'runtime_specs', - endpoint_response_times_histogram = 'endpoint_response_times_histogram', - endpoint_response_status = 'endpoint_response_status', - health_status = 'health_status', -} - -export enum HEALTH_STATUS { - Ok = 0, - StaleBulkAccountLoader, - UnhealthySlotSubscriber, - LivenessTesting, -} - -const metricsPort = - parseInt(process.env.METRICS_PORT) || PrometheusExporter.DEFAULT_OPTIONS.port; -const { endpoint: defaultEndpoint } = PrometheusExporter.DEFAULT_OPTIONS; -const exporter = new PrometheusExporter( - { - port: metricsPort, - endpoint: defaultEndpoint, - }, - () => { - logger.info( - `prometheus scrape endpoint started: http://localhost:${metricsPort}${defaultEndpoint}` - ); - } -); -const meterName = 'dlob-meter'; -const meterProvider = new MeterProvider({ - views: [ - new View({ - instrumentName: METRIC_TYPES.endpoint_response_times_histogram, - instrumentType: InstrumentType.HISTOGRAM, - meterName, - aggregation: createHistogramBuckets(0, 20, 30), - }), - ], -}); -meterProvider.addMetricReader(exporter); -const meter = meterProvider.getMeter(meterName); - -const runtimeSpecsGauge = meter.createObservableGauge( - METRIC_TYPES.runtime_specs, - { - description: 'Runtime sepcification of this program', - } -); -const bootTimeMs = Date.now(); -runtimeSpecsGauge.addCallback((obs) => { - obs.observe(bootTimeMs, { - commit: commitHash, - driftEnv, - rpcEndpoint: endpoint, - wsEndpoint: wsEndpoint, - }); -}); - -let healthStatus: HEALTH_STATUS = HEALTH_STATUS.Ok; -const healthStatusGauge = meter.createObservableGauge( - METRIC_TYPES.health_status, - { - description: 'Health status of this program', - } -); -healthStatusGauge.addCallback((obs: ObservableResult) => { - obs.observe(healthStatus, {}); -}); - -const endpointResponseTimeHistogram = meter.createHistogram( - METRIC_TYPES.endpoint_response_times_histogram, - { - description: 'Duration of endpoint responses', - unit: 'ms', - } -); - -const responseStatusCounter = meter.createCounter( - METRIC_TYPES.endpoint_response_status, - { - description: 'Count of endpoint responses by status code', - } -); - -const getPhoenixSubscriber = ( - driftClient: DriftClient, - marketConfig: SpotMarketConfig, - accountLoader: BulkAccountLoader -) => { - return new PhoenixSubscriber({ - connection: driftClient.connection, - programId: new PublicKey(sdkConfig.PHOENIX), - marketAddress: marketConfig.phoenixMarket, - accountSubscription: { - type: 'polling', - accountLoader, - }, - }); -}; - -const getSerumSubscriber = ( - driftClient: DriftClient, - marketConfig: SpotMarketConfig, - accountLoader: BulkAccountLoader -) => { - return new SerumSubscriber({ - connection: driftClient.connection, - programId: new PublicKey(sdkConfig.SERUM_V3), - marketAddress: marketConfig.serumMarket, - accountSubscription: { - type: 'polling', - accountLoader, - }, - }); -}; - -type SubscriberLookup = { - [marketIndex: number]: { - phoenix?: PhoenixSubscriber; - serum?: SerumSubscriber; - }; -}; - let MARKET_SUBSCRIBERS: SubscriberLookup = {}; -const initializeAllMarketSubscribers = async ( - driftClient: DriftClient, - bulkAccountLoader: BulkAccountLoader -) => { +const initializeAllMarketSubscribers = async (driftClient: DriftClient) => { const markets: SubscriberLookup = {}; for (const market of sdkConfig.SPOT_MARKETS) { @@ -317,21 +132,13 @@ const initializeAllMarketSubscribers = async ( }; if (market.phoenixMarket) { - const phoenixSubscriber = getPhoenixSubscriber( - driftClient, - market, - bulkAccountLoader - ); + const phoenixSubscriber = getPhoenixSubscriber(driftClient, market); await phoenixSubscriber.subscribe(); markets[market.marketIndex].phoenix = phoenixSubscriber; } if (market.serumMarket) { - const serumSubscriber = getSerumSubscriber( - driftClient, - market, - bulkAccountLoader - ); + const serumSubscriber = getSerumSubscriber(driftClient, market); await serumSubscriber.subscribe(); markets[market.marketIndex].serum = serumSubscriber; } @@ -349,21 +156,13 @@ const main = async () => { commitment: stateCommitment, }); - const bulkAccountLoader = new BulkAccountLoader( - connection, - stateCommitment, - bulkAccountLoaderPollingInterval - ); - const lastBulkAccountLoaderSlotMutex = new Mutex(); - let lastBulkAccountLoaderSlot = bulkAccountLoader.mostRecentSlot; - let lastBulkAccountLoaderSlotUpdated = Date.now(); - const driftClient = new DriftClient({ + driftClient = new DriftClient({ connection, wallet, programID: clearingHousePublicKey, accountSubscription: { - type: 'polling', - accountLoader: bulkAccountLoader, + type: 'websocket', + resubTimeoutMs: 60000, }, env: driftEnv, userStats: true, @@ -371,11 +170,6 @@ const main = async () => { const dlobCoder = DLOBOrdersCoder.create(); const slotSubscriber = new SlotSubscriber(connection, {}); - const lastSlotReceivedMutex = new Mutex(); - let lastSlotReceived: number; - let lastHealthCheckSlot = -1; - let lastHealthCheckSlotUpdated = Date.now(); - const startupTime = Date.now(); const lamportsBalance = await connection.getBalance(wallet.publicKey); logger.info( @@ -391,11 +185,6 @@ const main = async () => { }); await slotSubscriber.subscribe(); - slotSubscriber.eventEmitter.on('newSlot', async (slot: number) => { - await lastSlotReceivedMutex.runExclusive(async () => { - lastSlotReceived = slot; - }); - }); const userMap = new UserMap( driftClient, @@ -413,92 +202,11 @@ const main = async () => { driftClient, dlobSource: userMap, slotSource: slotSubscriber, - updateFrequency: bulkAccountLoaderPollingInterval, + updateFrequency: ORDERBOOK_UPDATE_INTERVAL, }); await dlobSubscriber.subscribe(); - MARKET_SUBSCRIBERS = await initializeAllMarketSubscribers( - driftClient, - bulkAccountLoader - ); - - const handleHealthCheck = async (req, res, next) => { - try { - if (req.url === '/health' || req.url === '/') { - if (opts.testLiveness) { - if (Date.now() > startupTime + 60 * 1000) { - healthStatus = HEALTH_STATUS.LivenessTesting; - - res.writeHead(500); - res.end('Testing liveness test fail'); - return; - } - } - // check if a slot was received recently - let healthySlotSubscriber = false; - await lastSlotReceivedMutex.runExclusive(async () => { - const slotChanged = lastSlotReceived > lastHealthCheckSlot; - const slotChangedRecently = - Date.now() - lastHealthCheckSlotUpdated < healthCheckInterval; - healthySlotSubscriber = slotChanged || slotChangedRecently; - logger.debug( - `Slotsubscriber health check: lastSlotReceived: ${lastSlotReceived}, lastHealthCheckSlot: ${lastHealthCheckSlot}, slotChanged: ${slotChanged}, slotChangedRecently: ${slotChangedRecently}` - ); - if (slotChanged) { - lastHealthCheckSlot = lastSlotReceived; - lastHealthCheckSlotUpdated = Date.now(); - } - }); - if (!healthySlotSubscriber) { - healthStatus = HEALTH_STATUS.UnhealthySlotSubscriber; - logger.error(`SlotSubscriber is not healthy`); - - res.writeHead(500); - res.end(`SlotSubscriber is not healthy`); - return; - } - - if (bulkAccountLoader) { - let healthyBulkAccountLoader = false; - await lastBulkAccountLoaderSlotMutex.runExclusive(async () => { - const slotChanged = - bulkAccountLoader.mostRecentSlot > lastBulkAccountLoaderSlot; - const slotChangedRecently = - Date.now() - lastBulkAccountLoaderSlotUpdated < - healthCheckInterval; - healthyBulkAccountLoader = slotChanged || slotChangedRecently; - logger.debug( - `BulkAccountLoader health check: bulkAccountLoader.mostRecentSlot: ${bulkAccountLoader.mostRecentSlot}, lastBulkAccountLoaderSlot: ${lastBulkAccountLoaderSlot}, slotChanged: ${slotChanged}, slotChangedRecently: ${slotChangedRecently}` - ); - if (slotChanged) { - lastBulkAccountLoaderSlot = bulkAccountLoader.mostRecentSlot; - lastBulkAccountLoaderSlotUpdated = Date.now(); - } - }); - if (!healthyBulkAccountLoader) { - healthStatus = HEALTH_STATUS.StaleBulkAccountLoader; - logger.error( - `Health check failed due to stale bulkAccountLoader.mostRecentSlot` - ); - - res.writeHead(501); - res.end(`bulkAccountLoader.mostRecentSlot is not healthy`); - return; - } - } - - // liveness check passed - healthStatus = HEALTH_STATUS.Ok; - res.writeHead(200); - res.end('OK'); - } else { - res.writeHead(404); - res.end('Not found'); - } - } catch (e) { - next(e); - } - }; + MARKET_SUBSCRIBERS = await initializeAllMarketSubscribers(driftClient); const handleStartup = async (_req, res, _next) => { if ( @@ -514,7 +222,6 @@ const main = async () => { } }; - // start http server listening to /health endpoint using http package app.get('/health', handleHealthCheck); app.get('/startup', handleStartup); app.get('/', handleHealthCheck); @@ -524,7 +231,7 @@ const main = async () => { // object with userAccount key and orders object serialized const orders: Array = []; const oracles: Array = []; - const slot = bulkAccountLoader.mostRecentSlot; + const slot = slotSubscriber.currentSlot; for (const market of driftClient.getPerpMarketAccounts()) { const oracle = driftClient.getOracleDataForPerpMarket( @@ -568,7 +275,7 @@ const main = async () => { app.get('/orders/json', async (_req, res, next) => { try { // object with userAccount key and orders object serialized - const slot = bulkAccountLoader.mostRecentSlot; + const slot = slotSubscriber.currentSlot; const orders: Array = []; const oracles: Array = []; for (const market of driftClient.getPerpMarketAccounts()) { @@ -729,7 +436,7 @@ const main = async () => { res.end( JSON.stringify({ - slot: bulkAccountLoader.mostRecentSlot, + slot: slotSubscriber.currentSlot, data: dlobCoder.encode(dlobOrders).toString('base64'), }) ); @@ -738,79 +445,6 @@ const main = async () => { } }); - const validateDlobQuery = ( - marketType?: string, - marketIndex?: string, - marketName?: string - ): { - normedMarketType?: MarketType; - normedMarketIndex?: number; - error?: string; - } => { - let normedMarketType: MarketType = undefined; - let normedMarketIndex: number = undefined; - let normedMarketName: string = undefined; - if (marketName === undefined) { - if (marketIndex === undefined || marketType === undefined) { - return { - error: - 'Bad Request: (marketName) or (marketIndex and marketType) must be supplied', - }; - } - - // validate marketType - switch ((marketType as string).toLowerCase()) { - case 'spot': { - normedMarketType = MarketType.SPOT; - normedMarketIndex = parseInt(marketIndex as string); - const spotMarketIndicies = SpotMarkets[driftEnv].map( - (mkt) => mkt.marketIndex - ); - if (!spotMarketIndicies.includes(normedMarketIndex)) { - return { - error: 'Bad Request: invalid marketIndex', - }; - } - break; - } - case 'perp': { - normedMarketType = MarketType.PERP; - normedMarketIndex = parseInt(marketIndex as string); - const perpMarketIndicies = PerpMarkets[driftEnv].map( - (mkt) => mkt.marketIndex - ); - if (!perpMarketIndicies.includes(normedMarketIndex)) { - return { - error: 'Bad Request: invalid marketIndex', - }; - } - break; - } - default: - return { - error: 'Bad Request: marketType must be either "spot" or "perp"', - }; - } - } else { - // validate marketName - normedMarketName = (marketName as string).toUpperCase(); - const derivedMarketInfo = - driftClient.getMarketIndexAndType(normedMarketName); - if (!derivedMarketInfo) { - return { - error: 'Bad Request: unrecognized marketName', - }; - } - normedMarketType = derivedMarketInfo.marketType; - normedMarketIndex = derivedMarketInfo.marketIndex; - } - - return { - normedMarketType, - normedMarketIndex, - }; - }; - app.get('/topMakers', async (req, res, next) => { try { const { @@ -914,21 +548,6 @@ const main = async () => { } }); - const getOracleForMarket = ( - marketType: MarketType, - marketIndex: number - ): number => { - if (isVariant(marketType, 'spot')) { - return driftClient - .getOracleDataForSpotMarket(marketIndex) - .price.toNumber(); - } else if (isVariant(marketType, 'perp')) { - return driftClient - .getOracleDataForPerpMarket(marketIndex) - .price.toNumber(); - } - }; - app.get('/l2', async (req, res, next) => { try { const { @@ -993,6 +612,7 @@ const main = async () => { ); if (`${includeOracle}`.toLowerCase() === 'true') { l2Formatted['oracle'] = getOracleForMarket( + driftClient, normedMarketType, normedMarketIndex ); @@ -1005,6 +625,7 @@ const main = async () => { const l2Formatted = l2WithBNToStrings(l2); if (`${includeOracle}`.toLowerCase() === 'true') { l2Formatted['oracle'] = getOracleForMarket( + driftClient, normedMarketType, normedMarketIndex ); @@ -1018,71 +639,6 @@ const main = async () => { } }); - /** - * Takes in a req.query like: `{ - * marketName: 'SOL-PERP,BTC-PERP,ETH-PERP', - * marketType: undefined, - * marketIndices: undefined, - * ... - * }` and returns a normalized object like: - * - * `[ - * {marketName: 'SOL-PERP', marketType: undefined, marketIndex: undefined,...}, - * {marketName: 'BTC-PERP', marketType: undefined, marketIndex: undefined,...}, - * {marketName: 'ETH-PERP', marketType: undefined, marketIndex: undefined,...} - * ]` - * - * @param rawParams req.query object - * @returns normalized query params for batch requests, or undefined if there is a mismatched length - */ - const normalizeBatchQueryParams = (rawParams: { - [key: string]: string | undefined; - }): Array<{ [key: string]: string | undefined }> => { - const normedParams: Array<{ [key: string]: string | undefined }> = []; - const parsedParams = {}; - - // parse the query string into arrays - for (const key of Object.keys(rawParams)) { - const rawParam = rawParams[key]; - if (rawParam === undefined) { - parsedParams[key] = []; - } else { - parsedParams[key] = rawParam.split(',') || [rawParam]; - } - } - - // of all parsedParams, find the max length - const maxLength = Math.max( - ...Object.values(parsedParams).map( - (param: Array) => param.length - ) - ); - - // all params have to be either 0 length, or maxLength to be valid - const values = Object.values(parsedParams); - const validParams = values.every( - (value: Array) => - value.length === 0 || value.length === maxLength - ); - if (!validParams) { - return undefined; - } - - // merge all params into an array of objects - // normalize all params to the same length, filling in undefineds - for (let i = 0; i < maxLength; i++) { - const newParam = {}; - for (const key of Object.keys(parsedParams)) { - const parsedParam = parsedParams[key]; - newParam[key] = - parsedParam.length === maxLength ? parsedParam[i] : undefined; - } - normedParams.push(newParam); - } - - return normedParams; - }; - app.get('/batchL2', async (req, res, next) => { try { const { @@ -1175,6 +731,7 @@ const main = async () => { ); if (`${normedParam['includeOracle']}`.toLowerCase() === 'true') { l2Formatted['oracle'] = getOracleForMarket( + driftClient, normedMarketType, normedMarketIndex ); @@ -1185,6 +742,7 @@ const main = async () => { const l2Formatted = l2WithBNToStrings(l2); if (`${normedParam['includeOracle']}`.toLowerCase() === 'true') { l2Formatted['oracle'] = getOracleForMarket( + driftClient, normedMarketType, normedMarketIndex ); @@ -1231,7 +789,11 @@ const main = async () => { } if (`${includeOracle}`.toLowerCase() === 'true') { - l3['oracle'] = getOracleForMarket(normedMarketType, normedMarketIndex); + l3['oracle'] = getOracleForMarket( + driftClient, + normedMarketType, + normedMarketIndex + ); } res.writeHead(200); @@ -1241,15 +803,10 @@ const main = async () => { } }); - app.use(errorHandler); server.listen(serverPort, () => { logger.info(`DLOB server listening on port http://localhost:${serverPort}`); }); - /** - * Handle websocket logic here - */ - io.on('connection', (socket) => { console.log('a user connected'); @@ -1260,7 +817,15 @@ const main = async () => { socket.on('message', (msg: any) => { const parsed = JSON.parse(msg); if (parsed['type'] == 'subscribe') { - console.log('here'); + const valid = validateWsSubscribeMsg(parsed); + if (!valid.valid) { + socket.emit('message', { + type: 'error', + message: `Bad Request: Invalid subscribe message: ${valid.msg}`, + }); + return; + } + dlobSubscriber.marketL2Args.push({ marketIndex: parseInt(parsed['marketIndex']), marketType: isVariant(parsed['marketType'].toLowerCase(), 'spot') @@ -1297,3 +862,5 @@ async function recursiveTryCatch(f: () => void) { } recursiveTryCatch(() => main()); + +export { sdkConfig, endpoint, wsEndpoint, driftEnv, commitHash, driftClient }; diff --git a/src/utils.ts b/src/utils.ts deleted file mode 100644 index 73d8fa1..0000000 --- a/src/utils.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { L2OrderBook } from '@drift-labs/sdk'; - -export const l2WithBNToStrings = (l2: L2OrderBook): any => { - for (const key of Object.keys(l2)) { - for (const idx in l2[key]) { - const level = l2[key][idx]; - const sources = level['sources']; - for (const sourceKey of Object.keys(sources)) { - sources[sourceKey] = sources[sourceKey].toString(); - } - l2[key][idx] = { - price: level.price.toString(), - size: level.size.toString(), - sources, - }; - } - } - return l2; -}; - -export function sleep(ms: number): Promise { - return new Promise((resolve) => setTimeout(resolve, ms)); -} diff --git a/src/logger.ts b/src/utils/logger.ts similarity index 100% rename from src/logger.ts rename to src/utils/logger.ts diff --git a/src/utils/utils.ts b/src/utils/utils.ts new file mode 100644 index 0000000..37e7936 --- /dev/null +++ b/src/utils/utils.ts @@ -0,0 +1,282 @@ +import { + DriftClient, + L2OrderBook, + MarketType, + PerpMarkets, + PhoenixSubscriber, + PublicKey, + SerumSubscriber, + SpotMarketConfig, + SpotMarkets, + isVariant, +} from '@drift-labs/sdk'; +import { logger } from './logger'; +import { NextFunction, Request, Response } from 'express'; +import { driftClient, driftEnv, sdkConfig } from '..'; + +export const l2WithBNToStrings = (l2: L2OrderBook): any => { + for (const key of Object.keys(l2)) { + for (const idx in l2[key]) { + const level = l2[key][idx]; + const sources = level['sources']; + for (const sourceKey of Object.keys(sources)) { + sources[sourceKey] = sources[sourceKey].toString(); + } + l2[key][idx] = { + price: level.price.toString(), + size: level.size.toString(), + sources, + }; + } + } + return l2; +}; + +export function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export const getOracleForMarket = ( + driftClient: DriftClient, + marketType: MarketType, + marketIndex: number +): number => { + if (isVariant(marketType, 'spot')) { + return driftClient.getOracleDataForSpotMarket(marketIndex).price.toNumber(); + } else if (isVariant(marketType, 'perp')) { + return driftClient.getOracleDataForPerpMarket(marketIndex).price.toNumber(); + } +}; + +/** + * Takes in a req.query like: `{ + * marketName: 'SOL-PERP,BTC-PERP,ETH-PERP', + * marketType: undefined, + * marketIndices: undefined, + * ... + * }` and returns a normalized object like: + * + * `[ + * {marketName: 'SOL-PERP', marketType: undefined, marketIndex: undefined,...}, + * {marketName: 'BTC-PERP', marketType: undefined, marketIndex: undefined,...}, + * {marketName: 'ETH-PERP', marketType: undefined, marketIndex: undefined,...} + * ]` + * + * @param rawParams req.query object + * @returns normalized query params for batch requests, or undefined if there is a mismatched length + */ +export const normalizeBatchQueryParams = (rawParams: { + [key: string]: string | undefined; +}): Array<{ [key: string]: string | undefined }> => { + const normedParams: Array<{ [key: string]: string | undefined }> = []; + const parsedParams = {}; + + // parse the query string into arrays + for (const key of Object.keys(rawParams)) { + const rawParam = rawParams[key]; + if (rawParam === undefined) { + parsedParams[key] = []; + } else { + parsedParams[key] = rawParam.split(',') || [rawParam]; + } + } + + // of all parsedParams, find the max length + const maxLength = Math.max( + ...Object.values(parsedParams).map((param: Array) => param.length) + ); + + // all params have to be either 0 length, or maxLength to be valid + const values = Object.values(parsedParams); + const validParams = values.every( + (value: Array) => value.length === 0 || value.length === maxLength + ); + if (!validParams) { + return undefined; + } + + // merge all params into an array of objects + // normalize all params to the same length, filling in undefineds + for (let i = 0; i < maxLength; i++) { + const newParam = {}; + for (const key of Object.keys(parsedParams)) { + const parsedParam = parsedParams[key]; + newParam[key] = + parsedParam.length === maxLength ? parsedParam[i] : undefined; + } + normedParams.push(newParam); + } + + return normedParams; +}; + +export const validateWsSubscribeMsg = ( + msg: any +): { valid: boolean; msg?: string } => { + const maxPerpMarketIndex = Math.max( + ...sdkConfig.PERP_MARKETS.map((m) => m.marketIndex) + ); + const maxSpotMarketIndex = Math.max( + ...sdkConfig.SPOT_MARKETS.map((m) => m.marketIndex) + ); + + if (msg['marketIndex'] < 0) { + return { valid: false, msg: `Invalid marketIndex, must be >= 0` }; + } + + if ( + msg['marketType'].toLowerCase() == 'spot' && + parseInt(msg['marketIndex']) > maxSpotMarketIndex + ) { + return { + valid: false, + msg: `Invalid marketIndex for marketType: ${msg['marketType']}`, + }; + } + + if ( + msg['marketType'].toLowerCase() == 'perp' && + parseInt(msg['marketIndex']) > maxPerpMarketIndex + ) { + return { + valid: false, + msg: `Invalid marketIndex for marketType: ${msg['marketType']}`, + }; + } + + if ( + msg['marketType'].toLowerCase() != 'perp' && + msg['marketType'] != 'spot' + ) { + return { + valid: false, + msg: `Invalid marketType: ${msg['marketType']}`, + }; + } + + return { valid: true }; +}; + +export const validateDlobQuery = ( + marketType?: string, + marketIndex?: string, + marketName?: string +): { + normedMarketType?: MarketType; + normedMarketIndex?: number; + error?: string; +} => { + let normedMarketType: MarketType = undefined; + let normedMarketIndex: number = undefined; + let normedMarketName: string = undefined; + if (marketName === undefined) { + if (marketIndex === undefined || marketType === undefined) { + return { + error: + 'Bad Request: (marketName) or (marketIndex and marketType) must be supplied', + }; + } + + // validate marketType + switch ((marketType as string).toLowerCase()) { + case 'spot': { + normedMarketType = MarketType.SPOT; + normedMarketIndex = parseInt(marketIndex as string); + const spotMarketIndicies = SpotMarkets[driftEnv].map( + (mkt) => mkt.marketIndex + ); + if (!spotMarketIndicies.includes(normedMarketIndex)) { + return { + error: 'Bad Request: invalid marketIndex', + }; + } + break; + } + case 'perp': { + normedMarketType = MarketType.PERP; + normedMarketIndex = parseInt(marketIndex as string); + const perpMarketIndicies = PerpMarkets[driftEnv].map( + (mkt) => mkt.marketIndex + ); + if (!perpMarketIndicies.includes(normedMarketIndex)) { + return { + error: 'Bad Request: invalid marketIndex', + }; + } + break; + } + default: + return { + error: 'Bad Request: marketType must be either "spot" or "perp"', + }; + } + } else { + // validate marketName + normedMarketName = (marketName as string).toUpperCase(); + const derivedMarketInfo = + driftClient.getMarketIndexAndType(normedMarketName); + if (!derivedMarketInfo) { + return { + error: 'Bad Request: unrecognized marketName', + }; + } + normedMarketType = derivedMarketInfo.marketType; + normedMarketIndex = derivedMarketInfo.marketIndex; + } + + return { + normedMarketType, + normedMarketIndex, + }; +}; + +export function errorHandler( + err: Error, + _req: Request, + res: Response, + _next: NextFunction +): void { + logger.error(`errorHandler, message: ${err.message}, stack: ${err.stack}`); + if (!res.headersSent) { + res.status(500).send('Internal error'); + } +} + +/** + * Spot market utils + */ + +export const getPhoenixSubscriber = ( + driftClient: DriftClient, + marketConfig: SpotMarketConfig +): PhoenixSubscriber => { + return new PhoenixSubscriber({ + connection: driftClient.connection, + programId: new PublicKey(sdkConfig.PHOENIX), + marketAddress: marketConfig.phoenixMarket, + accountSubscription: { + type: 'websocket', + }, + }); +}; + +export const getSerumSubscriber = ( + driftClient: DriftClient, + marketConfig: SpotMarketConfig +): SerumSubscriber => { + return new SerumSubscriber({ + connection: driftClient.connection, + programId: new PublicKey(sdkConfig.SERUM_V3), + marketAddress: marketConfig.serumMarket, + accountSubscription: { + type: 'websocket', + }, + }); +}; + +export type SubscriberLookup = { + [marketIndex: number]: { + phoenix?: PhoenixSubscriber; + serum?: SerumSubscriber; + }; +}; From de4d4c67591627e217d8d42e03611cdc2b465a28 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 25 Oct 2023 12:08:01 +0000 Subject: [PATCH 05/38] Bumping sdk dependency to 2.42.0-beta.17 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index ccf34ae..8ceadd9 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.42.0-beta.16", + "@drift-labs/sdk": "2.42.0-beta.17", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 9dc03d6..f8a479c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.42.0-beta.16": - version "2.42.0-beta.16" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.42.0-beta.16.tgz#cebc82306139c573c138742afd293c5466d5bdc3" - integrity sha512-kzhjiZF5Pas6I1pPVltjxz+H6b2Hi/7pCUscKrgl3Ot6mQ+UG2yorDJyWPIA89K18qPYKZ8+B5xC846bqgZUmQ== +"@drift-labs/sdk@2.42.0-beta.17": + version "2.42.0-beta.17" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.42.0-beta.17.tgz#9b44657831800fa14d476aaa69791a63909903d9" + integrity sha512-y3ZWe9gfKAX3VGDzpiXtwcMQkBgHD3co16BAJ+U8m5uEP5Wy8i+pG835/kQpjOj4pSjQgHBwMMmelg5EFcWW6Q== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 9bb5c9c062e78a0bf8e3c20635a80fa2909cde51 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Wed, 25 Oct 2023 11:15:46 -0700 Subject: [PATCH 06/38] publishing messages --- src/dlob-subscriber/DLOBSubscriberIO.ts | 42 ++++++++++++++++++++++++- src/index.ts | 40 +---------------------- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 6f1b586..adfbde1 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -3,6 +3,8 @@ import { DLOBSubscriber, DLOBSubscriptionConfig, L2OrderBookGenerator, + MainnetPerpMarkets, + MainnetSpotMarkets, MarketType, groupL2, } from '@drift-labs/sdk'; @@ -16,16 +18,49 @@ type wsMarketL2Args = { marketName: string; depth: number; includeVamm: boolean; + numVammOrders?: number; grouping?: number; fallbackL2Generators?: L2OrderBookGenerator[]; + updateOnChange?: boolean; }; export class DLOBSubscriberIO extends DLOBSubscriber { public marketL2Args: wsMarketL2Args[] = []; + public lastSeenL2Formatted: Map>; io: Server; constructor(config: DLOBSubscriptionConfig) { super(config); + + // Set up appropriate maps + this.lastSeenL2Formatted = new Map(); + this.lastSeenL2Formatted.set(MarketType.SPOT, new Map()); + this.lastSeenL2Formatted.set(MarketType.PERP, new Map()); + + // Add all active markets to the market L2Args + for (const market of MainnetPerpMarkets) { + this.marketL2Args.push({ + marketIndex: market.marketIndex, + marketType: MarketType.PERP, + marketName: market.symbol, + depth: 2, + includeVamm: true, + numVammOrders: 100, + updateOnChange: true, + fallbackL2Generators: [] + }); + } + for (const market of MainnetSpotMarkets) { + this.marketL2Args.push({ + marketIndex: market.marketIndex, + marketType: MarketType.SPOT, + marketName: market.symbol, + depth: 2, + includeVamm: false, + updateOnChange: true, + fallbackL2Generators: [] + }); + } } override async updateDLOB(): Promise { @@ -38,7 +73,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { getL2AndSendMsg(l2Args: wsMarketL2Args): void { const grouping = l2Args.grouping; - const l2 = this.getL2(l2Args); + const l2 = this.getL2(l2Args); let l2Formatted: any; if (grouping) { const groupingBN = new BN(grouping); @@ -46,6 +81,11 @@ export class DLOBSubscriberIO extends DLOBSubscriber { } else { l2Formatted = l2WithBNToStrings(l2); } + + if (l2Args.updateOnChange && this.lastSeenL2Formatted.get(l2Args.marketType)?.get(l2Args.marketIndex) === JSON.stringify(l2Formatted)) { + return; + } + this.lastSeenL2Formatted.get(l2Args.marketType)?.set(l2Args.marketIndex, JSON.stringify(l2Formatted)); l2Formatted['marketName'] = l2Args.marketName; l2Formatted['marketType'] = l2Args.marketType; l2Formatted['marketIndex'] = l2Args.marketIndex; diff --git a/src/index.ts b/src/index.ts index f4343a2..66d5216 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,7 +17,6 @@ import { DLOBOrder, DLOBOrders, DLOBOrdersCoder, - MarketType, DLOBNode, isVariant, BN, @@ -39,7 +38,6 @@ import { errorHandler, getPhoenixSubscriber, getSerumSubscriber, - validateWsSubscribeMsg, validateDlobQuery, } from './utils/utils'; import { DLOBSubscriberIO } from './dlob-subscriber/DLOBSubscriberIO'; @@ -54,7 +52,7 @@ const sdkConfig = initialize({ env: process.env.ENV }); const stateCommitment: Commitment = 'processed'; const serverPort = process.env.PORT || 6969; -const ORDERBOOK_UPDATE_INTERVAL = 100; +const ORDERBOOK_UPDATE_INTERVAL = 1000; const rateLimitCallsPerSecond = process.env.RATE_LIMIT_CALLS_PER_SECOND ? parseInt(process.env.RATE_LIMIT_CALLS_PER_SECOND) @@ -162,7 +160,6 @@ const main = async () => { programID: clearingHousePublicKey, accountSubscription: { type: 'websocket', - resubTimeoutMs: 60000, }, env: driftEnv, userStats: true, @@ -813,41 +810,6 @@ const main = async () => { socket.on('disconnect', () => { console.log('user disconnected'); }); - - socket.on('message', (msg: any) => { - const parsed = JSON.parse(msg); - if (parsed['type'] == 'subscribe') { - const valid = validateWsSubscribeMsg(parsed); - if (!valid.valid) { - socket.emit('message', { - type: 'error', - message: `Bad Request: Invalid subscribe message: ${valid.msg}`, - }); - return; - } - - dlobSubscriber.marketL2Args.push({ - marketIndex: parseInt(parsed['marketIndex']), - marketType: isVariant(parsed['marketType'].toLowerCase(), 'spot') - ? MarketType.SPOT - : MarketType.PERP, - marketName: parsed['marketName'], - depth: parseInt(parsed['depth']), - includeVamm: parsed['includeVamm'] === 'true', - grouping: parsed['grouping'] - ? parseInt(parsed['grouping']) - : undefined, - fallbackL2Generators: isVariant(parsed['marketType'], 'spot') - ? [ - `${parsed['includePhoenix']}`.toLowerCase() === 'true' && - MARKET_SUBSCRIBERS[parseInt(parsed['marketIndex'])].phoenix, - `${parsed['includeSerum']}`.toLowerCase() === 'true' && - MARKET_SUBSCRIBERS[parseInt(parsed['marketIndex'])].serum, - ].filter((a) => !!a) - : [], - }); - } - }); }); }; From 99cd43c01237101c75e67728d23f4c96a81e9d4c Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Wed, 25 Oct 2023 11:16:07 -0700 Subject: [PATCH 07/38] prettify --- src/dlob-subscriber/DLOBSubscriberIO.ts | 79 ++++++++++++++----------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index adfbde1..2b9dd03 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -18,49 +18,49 @@ type wsMarketL2Args = { marketName: string; depth: number; includeVamm: boolean; - numVammOrders?: number; + numVammOrders?: number; grouping?: number; fallbackL2Generators?: L2OrderBookGenerator[]; - updateOnChange?: boolean; + updateOnChange?: boolean; }; export class DLOBSubscriberIO extends DLOBSubscriber { public marketL2Args: wsMarketL2Args[] = []; - public lastSeenL2Formatted: Map>; + public lastSeenL2Formatted: Map>; io: Server; constructor(config: DLOBSubscriptionConfig) { super(config); - // Set up appropriate maps - this.lastSeenL2Formatted = new Map(); - this.lastSeenL2Formatted.set(MarketType.SPOT, new Map()); - this.lastSeenL2Formatted.set(MarketType.PERP, new Map()); + // Set up appropriate maps + this.lastSeenL2Formatted = new Map(); + this.lastSeenL2Formatted.set(MarketType.SPOT, new Map()); + this.lastSeenL2Formatted.set(MarketType.PERP, new Map()); - // Add all active markets to the market L2Args - for (const market of MainnetPerpMarkets) { - this.marketL2Args.push({ - marketIndex: market.marketIndex, - marketType: MarketType.PERP, - marketName: market.symbol, - depth: 2, - includeVamm: true, - numVammOrders: 100, - updateOnChange: true, - fallbackL2Generators: [] - }); - } - for (const market of MainnetSpotMarkets) { - this.marketL2Args.push({ - marketIndex: market.marketIndex, - marketType: MarketType.SPOT, - marketName: market.symbol, - depth: 2, - includeVamm: false, - updateOnChange: true, - fallbackL2Generators: [] - }); - } + // Add all active markets to the market L2Args + for (const market of MainnetPerpMarkets) { + this.marketL2Args.push({ + marketIndex: market.marketIndex, + marketType: MarketType.PERP, + marketName: market.symbol, + depth: 2, + includeVamm: true, + numVammOrders: 100, + updateOnChange: true, + fallbackL2Generators: [], + }); + } + for (const market of MainnetSpotMarkets) { + this.marketL2Args.push({ + marketIndex: market.marketIndex, + marketType: MarketType.SPOT, + marketName: market.symbol, + depth: 2, + includeVamm: false, + updateOnChange: true, + fallbackL2Generators: [], + }); + } } override async updateDLOB(): Promise { @@ -73,7 +73,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { getL2AndSendMsg(l2Args: wsMarketL2Args): void { const grouping = l2Args.grouping; - const l2 = this.getL2(l2Args); + const l2 = this.getL2(l2Args); let l2Formatted: any; if (grouping) { const groupingBN = new BN(grouping); @@ -82,10 +82,17 @@ export class DLOBSubscriberIO extends DLOBSubscriber { l2Formatted = l2WithBNToStrings(l2); } - if (l2Args.updateOnChange && this.lastSeenL2Formatted.get(l2Args.marketType)?.get(l2Args.marketIndex) === JSON.stringify(l2Formatted)) { - return; - } - this.lastSeenL2Formatted.get(l2Args.marketType)?.set(l2Args.marketIndex, JSON.stringify(l2Formatted)); + if ( + l2Args.updateOnChange && + this.lastSeenL2Formatted + .get(l2Args.marketType) + ?.get(l2Args.marketIndex) === JSON.stringify(l2Formatted) + ) { + return; + } + this.lastSeenL2Formatted + .get(l2Args.marketType) + ?.set(l2Args.marketIndex, JSON.stringify(l2Formatted)); l2Formatted['marketName'] = l2Args.marketName; l2Formatted['marketType'] = l2Args.marketType; l2Formatted['marketIndex'] = l2Args.marketIndex; From 7181912607cd41a6fd757bb32b0c2d98548ca078 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Wed, 25 Oct 2023 14:27:16 -0700 Subject: [PATCH 08/38] have redis working --- dump.rdb | Bin 0 -> 88 bytes example/newClient.ts | 0 package.json | 5 + src/dlob-subscriber/DLOBSubscriberIO.ts | 14 +- src/index.ts | 27 +++- src/utils/redisClient.ts | 124 ++++++++++++++++++ src/utils/utils.ts | 13 +- src/wsConnectionManager.ts | 49 +++++++ yarn.lock | 163 +++++++++++++++++++++++- 9 files changed, 384 insertions(+), 11 deletions(-) create mode 100644 dump.rdb create mode 100644 example/newClient.ts create mode 100644 src/utils/redisClient.ts create mode 100644 src/wsConnectionManager.ts diff --git a/dump.rdb b/dump.rdb new file mode 100644 index 0000000000000000000000000000000000000000..9c1c3154358e54599d7201eae61a0182d0600a19 GIT binary patch literal 88 zcmWG?b@2=~FfcUy#aWb^l3A=>; io: Server; + redisClient: RedisClient; - constructor(config: DLOBSubscriptionConfig) { + constructor(config: DLOBSubscriptionConfig & { redisClient: RedisClient }) { super(config); + this.redisClient = config.redisClient; // Set up appropriate maps this.lastSeenL2Formatted = new Map(); @@ -43,7 +46,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { marketIndex: market.marketIndex, marketType: MarketType.PERP, marketName: market.symbol, - depth: 2, + depth: -1, includeVamm: true, numVammOrders: 100, updateOnChange: true, @@ -55,7 +58,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { marketIndex: market.marketIndex, marketType: MarketType.SPOT, marketName: market.symbol, - depth: 2, + depth: -1, includeVamm: false, updateOnChange: true, fallbackL2Generators: [], @@ -101,6 +104,9 @@ export class DLOBSubscriberIO extends DLOBSubscriber { l2Args.marketType, l2Args.marketIndex ); - this.io.emit('l2Update', l2Formatted); + this.redisClient.client.publish( + l2Args.marketName, + JSON.stringify(l2Formatted) + ); } } diff --git a/src/index.ts b/src/index.ts index 66d5216..f0f132c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -43,6 +43,7 @@ import { import { DLOBSubscriberIO } from './dlob-subscriber/DLOBSubscriberIO'; import { handleResponseTime } from './core/middleware'; import { handleHealthCheck } from './core/metrics'; +import { RedisClient } from './utils/redisClient'; require('dotenv').config(); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; @@ -130,13 +131,21 @@ const initializeAllMarketSubscribers = async (driftClient: DriftClient) => { }; if (market.phoenixMarket) { - const phoenixSubscriber = getPhoenixSubscriber(driftClient, market); + const phoenixSubscriber = getPhoenixSubscriber( + driftClient, + market, + sdkConfig + ); await phoenixSubscriber.subscribe(); markets[market.marketIndex].phoenix = phoenixSubscriber; } if (market.serumMarket) { - const serumSubscriber = getSerumSubscriber(driftClient, market); + const serumSubscriber = getSerumSubscriber( + driftClient, + market, + sdkConfig + ); await serumSubscriber.subscribe(); markets[market.marketIndex].serum = serumSubscriber; } @@ -195,11 +204,15 @@ const main = async () => { ); await userStatsMap.subscribe(); + const redisClient = new RedisClient('localhost', '6379'); + await redisClient.connect(); + const dlobSubscriber = new DLOBSubscriberIO({ driftClient, dlobSource: userMap, slotSource: slotSubscriber, updateFrequency: ORDERBOOK_UPDATE_INTERVAL, + redisClient, }); await dlobSubscriber.subscribe(); @@ -385,6 +398,8 @@ const main = async () => { try { const { marketName, marketIndex, marketType } = req.query; const { normedMarketType, normedMarketIndex, error } = validateDlobQuery( + driftClient, + driftEnv, marketType as string, marketIndex as string, marketName as string @@ -454,6 +469,8 @@ const main = async () => { } = req.query; const { normedMarketType, normedMarketIndex, error } = validateDlobQuery( + driftClient, + driftEnv, marketType as string, marketIndex as string, marketName as string @@ -561,6 +578,8 @@ const main = async () => { } = req.query; const { normedMarketType, normedMarketIndex, error } = validateDlobQuery( + driftClient, + driftEnv, marketType as string, marketIndex as string, marketName as string @@ -674,6 +693,8 @@ const main = async () => { const l2s = normedParams.map((normedParam) => { const { normedMarketType, normedMarketIndex, error } = validateDlobQuery( + driftClient, + driftEnv, normedParam['marketType'] as string, normedParam['marketIndex'] as string, normedParam['marketName'] as string @@ -760,6 +781,8 @@ const main = async () => { const { marketName, marketIndex, marketType, includeOracle } = req.query; const { normedMarketType, normedMarketIndex, error } = validateDlobQuery( + driftClient, + driftEnv, marketType as string, marketIndex as string, marketName as string diff --git a/src/utils/redisClient.ts b/src/utils/redisClient.ts new file mode 100644 index 0000000..1bdf392 --- /dev/null +++ b/src/utils/redisClient.ts @@ -0,0 +1,124 @@ +import Redis, { RedisOptions } from 'ioredis'; +import { sleep } from './utils'; + +export const getRedisClient = ( + host?: string, + port?: string, + password?: string, + db?: number, + opts?: RedisOptions +): Redis => { + if (host && port) { + console.log(`Connecting to configured redis:: ${host}:${port}`); + + const redisClient = new Redis({ + host: host, + port: parseInt(port, 10), + password: password, + ...(opts ?? {}), + retryStrategy: (times) => { + const delay = Math.min(times * 1000, 10000); + console.log( + `Reconnecting to Redis in ${delay}ms... (retries: ${times})` + ); + return delay; + }, + reconnectOnError: (err) => { + const targetError = 'ECONNREFUSED'; + if (err.message.includes(targetError)) { + console.log( + `Redis error: ${targetError}. Attempting to reconnect...` + ); + return true; + } + return false; + }, + maxRetriesPerRequest: null, // unlimited retries + }); + + redisClient.on('connect', () => { + console.log('Connected to Redis.'); + }); + + redisClient.on('error', (err) => { + console.error('Redis error:', err); + }); + + redisClient.on('reconnecting', () => { + console.log('Reconnecting to Redis...'); + }); + + return redisClient; + } + + console.log(`Using default redis`); + return new Redis({ db }); +}; + +/** + * Wrapper around the redis client. + * + * You can hover over the underlying redis client methods for explanations of the methods, but will also include links to DOCS for some important concepts below: + * + * zRange, zRangeByScore etc.: + * - All of the "z" methods are methods that use sorted sets. + * - Sorted sets are explained here : https://redis.io/docs/data-types/sorted-sets/ + */ +export class RedisClient { + public client: Redis; + + connectionPromise: Promise; + + constructor( + host?: string, + port?: string, + password?: string, + db?: number, + opts?: RedisOptions + ) { + this.client = getRedisClient(host, port, password, db, opts); + } + + /** + * Should avoid using this unless necessary. + * @returns + */ + public forceGetClient() { + return this.client; + } + + public get connected() { + return this?.client?.status === 'ready'; + } + + async connect() { + if (this.client.status === 'ready' || this.client.status === 'connect') { + return; + } + + if (this.client.status === 'connecting') { + await sleep(100); + return this.connect(); // recursive call to check again + } + + try { + await this.client.connect(); + } catch (e) { + console.error(e); + } + } + + disconnect() { + this.client.disconnect(); + } + + private assertConnected() { + if (!this.connected) { + throw 'Redis client not connected'; + } + } + + private getPrefixForMetrics(key: string) { + return key.split(':')?.[0]; + } +} diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 37e7936..848de31 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -1,5 +1,6 @@ import { DriftClient, + DriftEnv, L2OrderBook, MarketType, PerpMarkets, @@ -12,7 +13,6 @@ import { } from '@drift-labs/sdk'; import { logger } from './logger'; import { NextFunction, Request, Response } from 'express'; -import { driftClient, driftEnv, sdkConfig } from '..'; export const l2WithBNToStrings = (l2: L2OrderBook): any => { for (const key of Object.keys(l2)) { @@ -111,7 +111,8 @@ export const normalizeBatchQueryParams = (rawParams: { }; export const validateWsSubscribeMsg = ( - msg: any + msg: any, + sdkConfig: any ): { valid: boolean; msg?: string } => { const maxPerpMarketIndex = Math.max( ...sdkConfig.PERP_MARKETS.map((m) => m.marketIndex) @@ -158,6 +159,8 @@ export const validateWsSubscribeMsg = ( }; export const validateDlobQuery = ( + driftClient: DriftClient, + driftEnv: DriftEnv, marketType?: string, marketIndex?: string, marketName?: string @@ -248,7 +251,8 @@ export function errorHandler( export const getPhoenixSubscriber = ( driftClient: DriftClient, - marketConfig: SpotMarketConfig + marketConfig: SpotMarketConfig, + sdkConfig ): PhoenixSubscriber => { return new PhoenixSubscriber({ connection: driftClient.connection, @@ -262,7 +266,8 @@ export const getPhoenixSubscriber = ( export const getSerumSubscriber = ( driftClient: DriftClient, - marketConfig: SpotMarketConfig + marketConfig: SpotMarketConfig, + sdkConfig ): SerumSubscriber => { return new SerumSubscriber({ connection: driftClient.connection, diff --git a/src/wsConnectionManager.ts b/src/wsConnectionManager.ts new file mode 100644 index 0000000..9734f80 --- /dev/null +++ b/src/wsConnectionManager.ts @@ -0,0 +1,49 @@ +import cors from 'cors'; +import express from 'express'; +import * as http from 'http'; +import { Server } from 'socket.io'; +import compression from 'compression'; +import { sleep } from './utils/utils'; +import { RedisClient } from './utils/redisClient'; + +const app = express(); +app.use(cors({ origin: '*' })); +app.use(compression()); +app.set('trust proxy', 1); + +const server = http.createServer(app); +const io = new Server(server); + +async function main() { + const redisClient = new RedisClient('localhost', '6379'); + await redisClient.connect(); + + io.on('connection', (socket) => { + socket.on('subscribe', (channel) => { + console.log('Subscribing to channel', channel); + redisClient.client.subscribe(channel); + redisClient.client.on('message', (subscribedChannel, message) => { + console.log(message); + if (subscribedChannel === channel) { + socket.emit(channel, JSON.parse(message)); + } + }); + }); + }); + + server.listen('3000', () => { + console.log('connection manager running on 3000'); + }); +} + +async function recursiveTryCatch(f: () => void) { + try { + await f(); + } catch (e) { + console.error(e); + await sleep(15000); + await recursiveTryCatch(f); + } +} + +recursiveTryCatch(() => main()); diff --git a/yarn.lock b/yarn.lock index 2195452..820677a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -220,6 +220,11 @@ resolved "https://registry.yarnpkg.com/@humanwhocodes/object-schema/-/object-schema-1.2.1.tgz#b520529ec21d8e5945a1851dfd1c32e94e39ff45" integrity sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA== +"@ioredis/commands@^1.1.1": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@ioredis/commands/-/commands-1.2.0.tgz#6d61b3097470af1fdbbe622795b8921d42018e11" + integrity sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg== + "@jridgewell/resolve-uri@^3.0.3": version "3.1.0" resolved "https://registry.yarnpkg.com/@jridgewell/resolve-uri/-/resolve-uri-3.1.0.tgz#2203b118c157721addfe69d47b70465463066d78" @@ -884,6 +889,40 @@ assert "^2.0.0" buffer "^6.0.1" +"@redis/bloom@1.2.0": + version "1.2.0" + resolved "https://registry.yarnpkg.com/@redis/bloom/-/bloom-1.2.0.tgz#d3fd6d3c0af3ef92f26767b56414a370c7b63b71" + integrity sha512-HG2DFjYKbpNmVXsa0keLHp/3leGJz1mjh09f2RLGGLQZzSHpkmZWuwJbAvo3QcRY8p80m5+ZdXZdYOSBLlp7Cg== + +"@redis/client@1.5.11": + version "1.5.11" + resolved "https://registry.yarnpkg.com/@redis/client/-/client-1.5.11.tgz#5ee8620fea56c67cb427228c35d8403518efe622" + integrity sha512-cV7yHcOAtNQ5x/yQl7Yw1xf53kO0FNDTdDU6bFIMbW6ljB7U7ns0YRM+QIkpoqTAt6zK5k9Fq0QWlUbLcq9AvA== + dependencies: + cluster-key-slot "1.1.2" + generic-pool "3.9.0" + yallist "4.0.0" + +"@redis/graph@1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@redis/graph/-/graph-1.1.0.tgz#cc2b82e5141a29ada2cce7d267a6b74baa6dd519" + integrity sha512-16yZWngxyXPd+MJxeSr0dqh2AIOi8j9yXKcKCwVaKDbH3HTuETpDVPcLujhFYVPtYrngSco31BUcSa9TH31Gqg== + +"@redis/json@1.0.6": + version "1.0.6" + resolved "https://registry.yarnpkg.com/@redis/json/-/json-1.0.6.tgz#b7a7725bbb907765d84c99d55eac3fcf772e180e" + integrity sha512-rcZO3bfQbm2zPRpqo82XbW8zg4G/w4W3tI7X8Mqleq9goQjAGLL7q/1n1ZX4dXEAmORVZ4s1+uKLaUOg7LrUhw== + +"@redis/search@1.1.5": + version "1.1.5" + resolved "https://registry.yarnpkg.com/@redis/search/-/search-1.1.5.tgz#682b68114049ff28fdf2d82c580044dfb74199fe" + integrity sha512-hPP8w7GfGsbtYEJdn4n7nXa6xt6hVZnnDktKW4ArMaFQ/m/aR7eFvsLQmG/mn1Upq99btPJk+F27IQ2dYpCoUg== + +"@redis/time-series@1.0.5": + version "1.0.5" + resolved "https://registry.yarnpkg.com/@redis/time-series/-/time-series-1.0.5.tgz#a6d70ef7a0e71e083ea09b967df0a0ed742bc6ad" + integrity sha512-IFjIgTusQym2B5IZJG3XKr5llka7ey84fw/NOYqESP5WUfQs9zz1ww/9+qoz4ka/S6KcGBodzlCeZ5UImKbscg== + "@sideway/address@^4.1.3": version "4.1.4" resolved "https://registry.yarnpkg.com/@sideway/address/-/address-4.1.4.tgz#03dccebc6ea47fdc226f7d3d1ad512955d4783f0" @@ -1356,6 +1395,13 @@ dependencies: "@types/node" "*" +"@types/redis@^4.0.11": + version "4.0.11" + resolved "https://registry.yarnpkg.com/@types/redis/-/redis-4.0.11.tgz#0bb4c11ac9900a21ad40d2a6768ec6aaf651c0e1" + integrity sha512-bI+gth8La8Wg/QCR1+V1fhrL9+LZUSWfcqpOj2Kc80ZQ4ffbdL173vQd5wovmoV9i071FU9oP2g6etLuEwb6Rg== + dependencies: + redis "*" + "@types/restify@4.3.8": version "4.3.8" resolved "https://registry.yarnpkg.com/@types/restify/-/restify-4.3.8.tgz#1e504995ef770a7149271e2cd639767cc7925245" @@ -1829,6 +1875,11 @@ chalk@^4.0.0: ansi-styles "^4.1.0" supports-color "^7.1.0" +cluster-key-slot@1.1.2, cluster-key-slot@^1.1.0: + version "1.1.2" + resolved "https://registry.yarnpkg.com/cluster-key-slot/-/cluster-key-slot-1.1.2.tgz#88ddaa46906e303b5de30d3153b7d9fe0a0c19ac" + integrity sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA== + color-convert@^1.9.0, color-convert@^1.9.3: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" @@ -2015,6 +2066,16 @@ delay@^5.0.0: resolved "https://registry.yarnpkg.com/delay/-/delay-5.0.0.tgz#137045ef1b96e5071060dd5be60bf9334436bd1d" integrity sha512-ReEBKkIfe4ya47wlPYf/gu5ib6yUG0/Aez0JQZQz94kiWtRQvZIQbTiehsnwHvLSWJnQdhVeqYue7Id1dKr0qw== +denque@^1.5.0: + version "1.5.1" + resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf" + integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw== + +denque@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/denque/-/denque-2.1.0.tgz#e93e1a6569fb5e66f16a3c2a2964617d349d6ab1" + integrity sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw== + depd@2.0.0, depd@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/depd/-/depd-2.0.0.tgz#b696163cc757560d09cf22cc8fad1571b79e76df" @@ -2568,6 +2629,11 @@ functional-red-black-tree@^1.0.1: resolved "https://registry.yarnpkg.com/functional-red-black-tree/-/functional-red-black-tree-1.0.1.tgz#1b0ab3bd553b2a0d6399d29c0e3ea0b252078327" integrity sha512-dsKNQNdj6xA3T+QlADDA7mOSlX0qiMINjn0cgr+eGHGsbSHzTabcIogz2+p/iqP1Xs6EP/sS2SbqH+brGTbq0g== +generic-pool@3.9.0: + version "3.9.0" + resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4" + integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g== + get-intrinsic@^1.0.2, get-intrinsic@^1.1.1, get-intrinsic@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.3.tgz#063c84329ad93e83893c7f4f243ef63ffa351385" @@ -2734,6 +2800,21 @@ inherits@2, inherits@2.0.4, inherits@^2.0.3: resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" integrity sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ== +ioredis@^5.3.2: + version "5.3.2" + resolved "https://registry.yarnpkg.com/ioredis/-/ioredis-5.3.2.tgz#9139f596f62fc9c72d873353ac5395bcf05709f7" + integrity sha512-1DKMMzlIHM02eBBVOFQ1+AolGjs6+xEcM4PDL7NqOS6szq7H9jSaEkIUH6/a5Hl241LzW6JLSiAbNvTQjUupUA== + dependencies: + "@ioredis/commands" "^1.1.1" + cluster-key-slot "^1.1.0" + debug "^4.3.4" + denque "^2.1.0" + lodash.defaults "^4.2.0" + lodash.isarguments "^3.1.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + standard-as-callback "^2.1.0" + ipaddr.js@1.9.1: version "1.9.1" resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.1.tgz#bff38543eeb8984825079ff3a2a8e6cbd46781b3" @@ -2928,6 +3009,16 @@ light-my-request@^4.2.0: process-warning "^1.0.0" set-cookie-parser "^2.4.1" +lodash.defaults@^4.2.0: + version "4.2.0" + resolved "https://registry.yarnpkg.com/lodash.defaults/-/lodash.defaults-4.2.0.tgz#d09178716ffea4dde9e5fb7b37f6f0802274580c" + integrity sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ== + +lodash.isarguments@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz#2f573d85c6a24289ff00663b491c1d338ff3458a" + integrity sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg== + lodash.merge@4.6.2, lodash.merge@^4.6.2: version "4.6.2" resolved "https://registry.yarnpkg.com/lodash.merge/-/lodash.merge-4.6.2.tgz#558aa53b43b661e1925a0afdfa36a9a1085fe57a" @@ -3093,6 +3184,11 @@ node-gyp-build@^4.3.0: resolved "https://registry.yarnpkg.com/node-gyp-build/-/node-gyp-build-4.5.0.tgz#7a64eefa0b21112f89f58379da128ac177f20e40" integrity sha512-2iGbaQBV+ITgCz76ZEjmhUKAKVf7xfY1sRl4UiKQspfZMH2h06SyhNsnSVy50cwkFQDGLyif6m/6uFXHkOZ6rg== +notepack.io@~2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/notepack.io/-/notepack.io-2.2.0.tgz#d7ea71d1cb90094f88c6f3c8d84277c2d0cd101c" + integrity sha512-9b5w3t5VSH6ZPosoYnyDONnUTF8o0UkBw7JLA6eBlYJWyGT1Q3vQa8Hmuj1/X6RYvHjjygBDgw6fJhe0JEojfw== + object-assign@^4: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" @@ -3395,6 +3491,45 @@ real-require@^0.1.0: resolved "https://registry.yarnpkg.com/real-require/-/real-require-0.1.0.tgz#736ac214caa20632847b7ca8c1056a0767df9381" integrity sha512-r/H9MzAWtrv8aSVjPCMFpDMl5q66GqtmmRkRjpHTsp4zBAa+snZyiQNlMONiUmEJcsnaw0wCauJ2GWODr/aFkg== +redis-commands@^1.7.0: + version "1.7.0" + resolved "https://registry.yarnpkg.com/redis-commands/-/redis-commands-1.7.0.tgz#15a6fea2d58281e27b1cd1acfb4b293e278c3a89" + integrity sha512-nJWqw3bTFy21hX/CPKHth6sfhZbdiHP6bTawSgQBlKOVRG7EZkfHbbHwQJnrE4vsQf0CMNE+3gJ4Fmm16vdVlQ== + +redis-errors@^1.0.0, redis-errors@^1.2.0: + version "1.2.0" + resolved "https://registry.yarnpkg.com/redis-errors/-/redis-errors-1.2.0.tgz#eb62d2adb15e4eaf4610c04afe1529384250abad" + integrity sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w== + +redis-parser@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/redis-parser/-/redis-parser-3.0.0.tgz#b66d828cdcafe6b4b8a428a7def4c6bcac31c8b4" + integrity sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A== + dependencies: + redis-errors "^1.0.0" + +redis@*, redis@^4.6.10: + version "4.6.10" + resolved "https://registry.yarnpkg.com/redis/-/redis-4.6.10.tgz#07f6ea2b2c5455b098e76d1e8c9b3376114e9458" + integrity sha512-mmbyhuKgDiJ5TWUhiKhBssz+mjsuSI/lSZNPI9QvZOYzWvYGejtb+W3RlDDf8LD6Bdl5/mZeG8O1feUGhXTxEg== + dependencies: + "@redis/bloom" "1.2.0" + "@redis/client" "1.5.11" + "@redis/graph" "1.1.0" + "@redis/json" "1.0.6" + "@redis/search" "1.1.5" + "@redis/time-series" "1.0.5" + +redis@^3.0.0: + version "3.1.2" + resolved "https://registry.yarnpkg.com/redis/-/redis-3.1.2.tgz#766851117e80653d23e0ed536254677ab647638c" + integrity sha512-grn5KoZLr/qrRQVwoSkmzdbw6pwF+/rwODtrOr6vuBRiR/f3rjSTGupbF90Zpqm2oenix8Do6RV7pYEkGwlKkw== + dependencies: + denque "^1.5.0" + redis-commands "^1.7.0" + redis-errors "^1.2.0" + redis-parser "^3.0.0" + regenerator-runtime@^0.13.11: version "0.13.11" resolved "https://registry.yarnpkg.com/regenerator-runtime/-/regenerator-runtime-0.13.11.tgz#f6dca3e7ceec20590d07ada785636a90cdca17f9" @@ -3641,6 +3776,11 @@ snake-case@^3.0.4: dot-case "^3.0.4" tslib "^2.0.3" +socket.io-adapter@~2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-2.2.0.tgz#43af9157c4609e74b8addc6867873ac7eb48fda2" + integrity sha512-rG49L+FwaVEwuAdeBRq49M97YI3ElVabJPzvHT9S6a2CWhDKnjSFasvwAwSYPRhQzfn4NtDIbCaGYgOCOU/rlg== + socket.io-adapter@~2.5.2: version "2.5.2" resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-2.5.2.tgz#5de9477c9182fdc171cd8c8364b9a8894ec75d12" @@ -3666,6 +3806,17 @@ socket.io-parser@~4.2.4: "@socket.io/component-emitter" "~3.1.0" debug "~4.3.1" +socket.io-redis@^6.1.1: + version "6.1.1" + resolved "https://registry.yarnpkg.com/socket.io-redis/-/socket.io-redis-6.1.1.tgz#2361029a6c0b25c602d1422e1beb41907fd0e8bf" + integrity sha512-jeaXe3TGKC20GMSlPHEdwTUIWUpay/L7m5+S9TQcOf22p9Llx44/RkpJV08+buXTZ8E+aivOotj2RdeFJJWJJQ== + dependencies: + debug "~4.3.1" + notepack.io "~2.2.0" + redis "^3.0.0" + socket.io-adapter "~2.2.0" + uid2 "0.0.3" + socket.io@^4.7.2: version "4.7.2" resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.7.2.tgz#22557d76c3f3ca48f82e73d68b7add36a22df002" @@ -3717,6 +3868,11 @@ stack-trace@0.0.x: resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" integrity sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg== +standard-as-callback@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/standard-as-callback/-/standard-as-callback-2.1.0.tgz#8953fc05359868a77b5b9739a665c5977bb7df45" + integrity sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A== + statuses@2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/statuses/-/statuses-2.0.1.tgz#55cb000ccf1d48728bd23c685a063998cf1a1b63" @@ -3930,6 +4086,11 @@ typescript@4.5.4: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.5.4.tgz#a17d3a0263bf5c8723b9c52f43c5084edf13c2e8" integrity sha512-VgYs2A2QIRuGphtzFV7aQJduJ2gyfTljngLzjpfW9FoYZF6xuw1W0vW9ghCKLfcWrCFxK81CSGRAvS1pn4fIUg== +uid2@0.0.3: + version "0.0.3" + resolved "https://registry.yarnpkg.com/uid2/-/uid2-0.0.3.tgz#483126e11774df2f71b8b639dcd799c376162b82" + integrity sha512-5gSP1liv10Gjp8cMEnFd6shzkL/D6W1uhXSFNCxDC+YI8+L8wkCYCbJ7n77Ezb4wE/xzMogecE+DtamEe9PZjg== + undici-types@~5.25.1: version "5.25.3" resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.25.3.tgz#e044115914c85f0bcbb229f346ab739f064998c3" @@ -4083,7 +4244,7 @@ xtend@^4.0.0: resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" integrity sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ== -yallist@^4.0.0: +yallist@4.0.0, yallist@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72" integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A== From 920b03d8465f9db5f60c131e9dc11b29625af115 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:06:14 +0000 Subject: [PATCH 09/38] Bumping sdk dependency to 2.42.0-beta.18 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 8ceadd9..e06ecf4 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.42.0-beta.17", + "@drift-labs/sdk": "2.42.0-beta.18", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index f8a479c..7a3273e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.42.0-beta.17": - version "2.42.0-beta.17" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.42.0-beta.17.tgz#9b44657831800fa14d476aaa69791a63909903d9" - integrity sha512-y3ZWe9gfKAX3VGDzpiXtwcMQkBgHD3co16BAJ+U8m5uEP5Wy8i+pG835/kQpjOj4pSjQgHBwMMmelg5EFcWW6Q== +"@drift-labs/sdk@2.42.0-beta.18": + version "2.42.0-beta.18" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.42.0-beta.18.tgz#40150612b673e8e76a163e57e786122ad6839d0e" + integrity sha512-acV4tGkPeBrzko4WFzBg3jskGrC/KU86Z+K7vMXGhTi0TO4bPF92pl5ZozMJ5LhTnj3c78nPMmqSme+Gm2d2Zw== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From e047def3a6a1564bee9ac48e88e88b618c6e97e3 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Thu, 26 Oct 2023 08:25:17 -0700 Subject: [PATCH 10/38] newclient --- .gitignore | 2 ++ example/newClient.ts | 17 +++++++++++++++++ src/dlob-subscriber/DLOBSubscriberIO.ts | 9 ++++----- src/wsConnectionManager.ts | 1 - 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 61ce045..3f820bc 100644 --- a/.gitignore +++ b/.gitignore @@ -141,3 +141,5 @@ src/**.js.map .env lib src/playground.ts + +dump.rdb \ No newline at end of file diff --git a/example/newClient.ts b/example/newClient.ts index e69de29..913549e 100644 --- a/example/newClient.ts +++ b/example/newClient.ts @@ -0,0 +1,17 @@ +import { io } from 'socket.io-client'; + +const main = async () => { + const socket = io('http://localhost:3000'); + + socket.on('connect', () => { + socket.emit('subscribe', 'SOL-PERP'); + }); + + socket.on('SOL-PERP', (data: any) => { + console.log(data); + }); +}; + +main().then(() => { + console.log('running'); +}); diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index fac3149..f0d4682 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -86,12 +86,11 @@ export class DLOBSubscriberIO extends DLOBSubscriber { } if ( - l2Args.updateOnChange && - this.lastSeenL2Formatted - .get(l2Args.marketType) - ?.get(l2Args.marketIndex) === JSON.stringify(l2Formatted) + l2Args.updateOnChange ) { - return; + if (this.lastSeenL2Formatted + .get(l2Args.marketType) + ?.get(l2Args.marketIndex) === JSON.stringify(l2Formatted)) return; } this.lastSeenL2Formatted .get(l2Args.marketType) diff --git a/src/wsConnectionManager.ts b/src/wsConnectionManager.ts index 9734f80..e5120ac 100644 --- a/src/wsConnectionManager.ts +++ b/src/wsConnectionManager.ts @@ -23,7 +23,6 @@ async function main() { console.log('Subscribing to channel', channel); redisClient.client.subscribe(channel); redisClient.client.on('message', (subscribedChannel, message) => { - console.log(message); if (subscribedChannel === channel) { socket.emit(channel, JSON.parse(message)); } From 2cf26d109e4cebb7f7f817e5403705852608acfa Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 26 Oct 2023 22:24:10 +0000 Subject: [PATCH 11/38] Bumping sdk dependency to 2.43.0-beta.0 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index e06ecf4..6a4eed1 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.42.0-beta.18", + "@drift-labs/sdk": "2.43.0-beta.0", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 7a3273e..f595f44 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.42.0-beta.18": - version "2.42.0-beta.18" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.42.0-beta.18.tgz#40150612b673e8e76a163e57e786122ad6839d0e" - integrity sha512-acV4tGkPeBrzko4WFzBg3jskGrC/KU86Z+K7vMXGhTi0TO4bPF92pl5ZozMJ5LhTnj3c78nPMmqSme+Gm2d2Zw== +"@drift-labs/sdk@2.43.0-beta.0": + version "2.43.0-beta.0" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.0.tgz#43da769f9eaf47ecb455e44f4cd09b5aaa3062d1" + integrity sha512-iYXVJdPn3RTqMsXw/HHqztQVLeZtC4XOlb4jMwTeuXsP3/xfkWOXJjlCOM51PWD2iVXu6sG67MgST4ipsXtjew== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 63d85ddc6717000afc2e699c225bf8873eb76414 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Fri, 27 Oct 2023 09:45:43 -0700 Subject: [PATCH 12/38] separate out publisher --- dump.rdb | Bin 88 -> 0 bytes package.json | 1 + src/dlob-subscriber/DLOBSubscriberIO.ts | 17 ++-- src/index.ts | 23 +---- src/wsConnectionManager.ts | 5 +- src/wsPublish.ts | 121 ++++++++++++++++++++++++ 6 files changed, 135 insertions(+), 32 deletions(-) delete mode 100644 dump.rdb create mode 100644 src/wsPublish.ts diff --git a/dump.rdb b/dump.rdb deleted file mode 100644 index 9c1c3154358e54599d7201eae61a0182d0600a19..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 88 zcmWG?b@2=~FfcUy#aWb^l3A=>; - io: Server; redisClient: RedisClient; constructor(config: DLOBSubscriptionConfig & { redisClient: RedisClient }) { @@ -67,7 +64,6 @@ export class DLOBSubscriberIO extends DLOBSubscriber { } override async updateDLOB(): Promise { - this.io = getIOServer(); await super.updateDLOB(); for (const l2Args of this.marketL2Args) { this.getL2AndSendMsg(l2Args); @@ -85,12 +81,13 @@ export class DLOBSubscriberIO extends DLOBSubscriber { l2Formatted = l2WithBNToStrings(l2); } - if ( - l2Args.updateOnChange - ) { - if (this.lastSeenL2Formatted - .get(l2Args.marketType) - ?.get(l2Args.marketIndex) === JSON.stringify(l2Formatted)) return; + if (l2Args.updateOnChange) { + if ( + this.lastSeenL2Formatted + .get(l2Args.marketType) + ?.get(l2Args.marketIndex) === JSON.stringify(l2Formatted) + ) + return; } this.lastSeenL2Formatted .get(l2Args.marketType) diff --git a/src/index.ts b/src/index.ts index f0f132c..891e3ac 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,12 +23,12 @@ import { groupL2, Wallet, UserStatsMap, + DLOBSubscriber, } from '@drift-labs/sdk'; import { logger, setLogLevel } from './utils/logger'; import * as http from 'http'; -import { Server } from 'socket.io'; import { l2WithBNToStrings, sleep, @@ -40,10 +40,8 @@ import { getSerumSubscriber, validateDlobQuery, } from './utils/utils'; -import { DLOBSubscriberIO } from './dlob-subscriber/DLOBSubscriberIO'; import { handleResponseTime } from './core/middleware'; import { handleHealthCheck } from './core/metrics'; -import { RedisClient } from './utils/redisClient'; require('dotenv').config(); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; @@ -103,11 +101,6 @@ app.use((req, _res, next) => { app.use(errorHandler); const server = http.createServer(app); -const io = new Server(server); - -export function getIOServer(): Server { - return io; -} const opts = program.opts(); setLogLevel(opts.debug ? 'debug' : 'info'); @@ -204,15 +197,11 @@ const main = async () => { ); await userStatsMap.subscribe(); - const redisClient = new RedisClient('localhost', '6379'); - await redisClient.connect(); - - const dlobSubscriber = new DLOBSubscriberIO({ + const dlobSubscriber = new DLOBSubscriber({ driftClient, dlobSource: userMap, slotSource: slotSubscriber, updateFrequency: ORDERBOOK_UPDATE_INTERVAL, - redisClient, }); await dlobSubscriber.subscribe(); @@ -826,14 +815,6 @@ const main = async () => { server.listen(serverPort, () => { logger.info(`DLOB server listening on port http://localhost:${serverPort}`); }); - - io.on('connection', (socket) => { - console.log('a user connected'); - - socket.on('disconnect', () => { - console.log('user disconnected'); - }); - }); }; async function recursiveTryCatch(f: () => void) { diff --git a/src/wsConnectionManager.ts b/src/wsConnectionManager.ts index e5120ac..1faf185 100644 --- a/src/wsConnectionManager.ts +++ b/src/wsConnectionManager.ts @@ -14,8 +14,11 @@ app.set('trust proxy', 1); const server = http.createServer(app); const io = new Server(server); +const REDIS_HOST = process.env.REDIS_HOST || 'localhost'; +const REDIS_PORT = process.env.REDIS_PORT || '6379'; + async function main() { - const redisClient = new RedisClient('localhost', '6379'); + const redisClient = new RedisClient(REDIS_HOST, REDIS_PORT); await redisClient.connect(); io.on('connection', (socket) => { diff --git a/src/wsPublish.ts b/src/wsPublish.ts new file mode 100644 index 0000000..c814e33 --- /dev/null +++ b/src/wsPublish.ts @@ -0,0 +1,121 @@ +import { program } from 'commander'; + +import { Connection, Commitment, PublicKey, Keypair } from '@solana/web3.js'; + +import { + DriftClient, + initialize, + DriftEnv, + SlotSubscriber, + UserMap, + Wallet, + UserStatsMap, +} from '@drift-labs/sdk'; + +import { logger, setLogLevel } from './utils/logger'; + +import { sleep } from './utils/utils'; +import { DLOBSubscriberIO } from './dlob-subscriber/DLOBSubscriberIO'; +import { RedisClient } from './utils/redisClient'; + +require('dotenv').config(); +const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; +const commitHash = process.env.COMMIT; +const REDIS_HOST = process.env.REDIS_HOST || 'localhost'; +const REDIS_PORT = process.env.REDIS_PORT || '6379'; + +//@ts-ignore +const sdkConfig = initialize({ env: process.env.ENV }); + +const stateCommitment: Commitment = 'processed'; +const ORDERBOOK_UPDATE_INTERVAL = 1000; + +let driftClient: DriftClient; + +const opts = program.opts(); +setLogLevel(opts.debug ? 'debug' : 'info'); + +const endpoint = process.env.ENDPOINT; +const wsEndpoint = process.env.WS_ENDPOINT; +logger.info(`RPC endpoint: ${endpoint}`); +logger.info(`WS endpoint: ${wsEndpoint}`); +logger.info(`DriftEnv: ${driftEnv}`); +logger.info(`Commit: ${commitHash}`); + +const main = async () => { + const wallet = new Wallet(new Keypair()); + const clearingHousePublicKey = new PublicKey(sdkConfig.DRIFT_PROGRAM_ID); + + const connection = new Connection(endpoint, { + wsEndpoint: wsEndpoint, + commitment: stateCommitment, + }); + + driftClient = new DriftClient({ + connection, + wallet, + programID: clearingHousePublicKey, + accountSubscription: { + type: 'websocket', + }, + env: driftEnv, + userStats: true, + }); + + const slotSubscriber = new SlotSubscriber(connection, {}); + + const lamportsBalance = await connection.getBalance(wallet.publicKey); + logger.info( + `DriftClient ProgramId: ${driftClient.program.programId.toBase58()}` + ); + logger.info(`Wallet pubkey: ${wallet.publicKey.toBase58()}`); + logger.info(` . SOL balance: ${lamportsBalance / 10 ** 9}`); + + await driftClient.subscribe(); + driftClient.eventEmitter.on('error', (e) => { + logger.info('clearing house error'); + logger.error(e); + }); + + await slotSubscriber.subscribe(); + + const userMap = new UserMap( + driftClient, + driftClient.userAccountSubscriptionConfig, + false + ); + await userMap.subscribe(); + const userStatsMap = new UserStatsMap( + driftClient, + driftClient.userStatsAccountSubscriptionConfig + ); + await userStatsMap.subscribe(); + + const redisClient = new RedisClient(REDIS_HOST, REDIS_PORT); + await redisClient.connect(); + + const dlobSubscriber = new DLOBSubscriberIO({ + driftClient, + dlobSource: userMap, + slotSource: slotSubscriber, + updateFrequency: ORDERBOOK_UPDATE_INTERVAL, + redisClient, + }); + await dlobSubscriber.subscribe(); + + console.log('DLOBSubscriber Publishing Messages'); +}; + +async function recursiveTryCatch(f: () => void) { + try { + await f(); + } catch (e) { + console.error(e); + await sleep(15000); + await recursiveTryCatch(f); + } +} + +recursiveTryCatch(() => main()); + +export { sdkConfig, endpoint, wsEndpoint, driftEnv, commitHash, driftClient }; From b6237770672042ce9335489ef5dd4b28c86788ec Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Fri, 27 Oct 2023 11:39:05 -0700 Subject: [PATCH 13/38] better health check --- .gitignore | 2 +- src/core/metrics.ts | 36 +++++++++++++++++++++++++++++++----- src/index.ts | 14 ++++++++++++++ 3 files changed, 46 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 3f820bc..acde1c6 100644 --- a/.gitignore +++ b/.gitignore @@ -142,4 +142,4 @@ src/**.js.map lib src/playground.ts -dump.rdb \ No newline at end of file +*dump.rdb \ No newline at end of file diff --git a/src/core/metrics.ts b/src/core/metrics.ts index 4eaabca..6032d9e 100644 --- a/src/core/metrics.ts +++ b/src/core/metrics.ts @@ -7,7 +7,13 @@ import { MeterProvider, View, } from '@opentelemetry/sdk-metrics-base'; -import { commitHash, driftEnv, endpoint, wsEndpoint } from '..'; +import { + commitHash, + driftEnv, + endpoint, + getSlotHealthCheckInfo, + wsEndpoint, +} from '..'; /** * Creates {count} buckets of size {increment} starting from {start}. Each bucket stores the count of values within its "size". @@ -110,15 +116,35 @@ const responseStatusCounter = meter.createCounter( } ); -const startupTime = Date.now(); +const healthCheckInterval = 2000; +let lastHealthCheckSlot = -1; +let lastHealthCheckSlotUpdated = Date.now(); const handleHealthCheck = async (req, res, next) => { + const { lastSlotReceived, lastSlotReceivedMutex } = getSlotHealthCheckInfo(); + try { if (req.url === '/health' || req.url === '/') { - if (Date.now() > startupTime + 60 * 1000) { - healthStatus = HEALTH_STATUS.LivenessTesting; + // check if a slot was received recently + let healthySlotSubscriber = false; + await lastSlotReceivedMutex.runExclusive(async () => { + const slotChanged = lastSlotReceived > lastHealthCheckSlot; + const slotChangedRecently = + Date.now() - lastHealthCheckSlotUpdated < healthCheckInterval; + healthySlotSubscriber = slotChanged || slotChangedRecently; + logger.debug( + `Slotsubscriber health check: lastSlotReceived: ${lastSlotReceived}, lastHealthCheckSlot: ${lastHealthCheckSlot}, slotChanged: ${slotChanged}, slotChangedRecently: ${slotChangedRecently}` + ); + if (slotChanged) { + lastHealthCheckSlot = lastSlotReceived; + lastHealthCheckSlotUpdated = Date.now(); + } + }); + if (!healthySlotSubscriber) { + healthStatus = HEALTH_STATUS.UnhealthySlotSubscriber; + logger.error(`SlotSubscriber is not healthy`); res.writeHead(500); - res.end('Testing liveness test fail'); + res.end(`SlotSubscriber is not healthy`); return; } diff --git a/src/index.ts b/src/index.ts index 891e3ac..e7d1a5a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -42,6 +42,7 @@ import { } from './utils/utils'; import { handleResponseTime } from './core/middleware'; import { handleHealthCheck } from './core/metrics'; +import { Mutex } from 'async-mutex'; require('dotenv').config(); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; @@ -114,6 +115,13 @@ logger.info(`Commit: ${commitHash}`); let MARKET_SUBSCRIBERS: SubscriberLookup = {}; +const lastSlotReceivedMutex = new Mutex(); +let lastSlotReceived: number; + +export const getSlotHealthCheckInfo = () => { + return { lastSlotReceived, lastSlotReceivedMutex }; +}; + const initializeAllMarketSubscribers = async (driftClient: DriftClient) => { const markets: SubscriberLookup = {}; @@ -162,6 +170,7 @@ const main = async () => { programID: clearingHousePublicKey, accountSubscription: { type: 'websocket', + resubTimeoutMs: 60000, }, env: driftEnv, userStats: true, @@ -184,6 +193,11 @@ const main = async () => { }); await slotSubscriber.subscribe(); + slotSubscriber.eventEmitter.on('newSlot', async (slot: number) => { + await lastSlotReceivedMutex.runExclusive(async () => { + lastSlotReceived = slot; + }); + }); const userMap = new UserMap( driftClient, From bf472e4c6b81d2d0f85d1f73d18a670c12dd0e23 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 27 Oct 2023 19:02:58 +0000 Subject: [PATCH 14/38] Bumping sdk dependency to 2.43.0-beta.1 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 2a50e9c..535cd2f 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.43.0-beta.0", + "@drift-labs/sdk": "2.43.0-beta.1", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 45321d9..6973f25 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.43.0-beta.0": - version "2.43.0-beta.0" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.0.tgz#43da769f9eaf47ecb455e44f4cd09b5aaa3062d1" - integrity sha512-iYXVJdPn3RTqMsXw/HHqztQVLeZtC4XOlb4jMwTeuXsP3/xfkWOXJjlCOM51PWD2iVXu6sG67MgST4ipsXtjew== +"@drift-labs/sdk@2.43.0-beta.1": + version "2.43.0-beta.1" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.1.tgz#b1cc01fa5f1bdba6c4597b783f75cc24cf74909e" + integrity sha512-Ij0+C2d3bhoLhAMNqy4omqe8tN+CSAAOVUrgVCBNFr9LHwmUFITJLt11vpj355FdJxH1frR7EW2s4M3lZdsrog== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 2eb723415815c7ab98b6b4435cd291ca9b83db0f Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Fri, 27 Oct 2023 13:47:39 -0700 Subject: [PATCH 15/38] including redis password for publisher --- src/wsPublish.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/wsPublish.ts b/src/wsPublish.ts index c814e33..f7d73be 100644 --- a/src/wsPublish.ts +++ b/src/wsPublish.ts @@ -23,6 +23,7 @@ const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; const commitHash = process.env.COMMIT; const REDIS_HOST = process.env.REDIS_HOST || 'localhost'; const REDIS_PORT = process.env.REDIS_PORT || '6379'; +const REDIS_PASSWORD = process.env.REDIS_PASSWORD; //@ts-ignore const sdkConfig = initialize({ env: process.env.ENV }); @@ -91,7 +92,7 @@ const main = async () => { ); await userStatsMap.subscribe(); - const redisClient = new RedisClient(REDIS_HOST, REDIS_PORT); + const redisClient = new RedisClient(REDIS_HOST, REDIS_PORT, REDIS_PASSWORD); await redisClient.connect(); const dlobSubscriber = new DLOBSubscriberIO({ From 87d79abb576a7bf11b70b208c6ddf4ff9e99fef7 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Fri, 27 Oct 2023 14:41:13 -0700 Subject: [PATCH 16/38] remove user stats map from ws publish --- src/wsPublish.ts | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/wsPublish.ts b/src/wsPublish.ts index f7d73be..977163a 100644 --- a/src/wsPublish.ts +++ b/src/wsPublish.ts @@ -9,11 +9,9 @@ import { SlotSubscriber, UserMap, Wallet, - UserStatsMap, } from '@drift-labs/sdk'; import { logger, setLogLevel } from './utils/logger'; - import { sleep } from './utils/utils'; import { DLOBSubscriberIO } from './dlob-subscriber/DLOBSubscriberIO'; import { RedisClient } from './utils/redisClient'; @@ -86,11 +84,6 @@ const main = async () => { false ); await userMap.subscribe(); - const userStatsMap = new UserStatsMap( - driftClient, - driftClient.userStatsAccountSubscriptionConfig - ); - await userStatsMap.subscribe(); const redisClient = new RedisClient(REDIS_HOST, REDIS_PORT, REDIS_PASSWORD); await redisClient.connect(); From d907d9869f9594d304a91a704da90f850d475680 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Fri, 27 Oct 2023 15:03:41 -0700 Subject: [PATCH 17/38] dlobsubscriberio can do devnet or mainnet --- src/dlob-subscriber/DLOBSubscriberIO.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 188dd62..d371e75 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -2,6 +2,8 @@ import { BN, DLOBSubscriber, DLOBSubscriptionConfig, + DevnetPerpMarkets, + DevnetSpotMarkets, L2OrderBookGenerator, MainnetPerpMarkets, MainnetSpotMarkets, @@ -10,6 +12,7 @@ import { } from '@drift-labs/sdk'; import { getOracleForMarket, l2WithBNToStrings } from '../utils/utils'; import { RedisClient } from '../utils/redisClient'; +import { driftEnv } from '../wsPublish'; type wsMarketL2Args = { marketIndex: number; @@ -38,7 +41,12 @@ export class DLOBSubscriberIO extends DLOBSubscriber { this.lastSeenL2Formatted.set(MarketType.PERP, new Map()); // Add all active markets to the market L2Args - for (const market of MainnetPerpMarkets) { + const perpMarkets = + driftEnv === 'devnet' ? DevnetPerpMarkets : MainnetPerpMarkets; + const spotMarkets = + driftEnv === 'devnet' ? DevnetSpotMarkets : MainnetSpotMarkets; + + for (const market of perpMarkets) { this.marketL2Args.push({ marketIndex: market.marketIndex, marketType: MarketType.PERP, @@ -50,7 +58,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { fallbackL2Generators: [], }); } - for (const market of MainnetSpotMarkets) { + for (const market of spotMarkets) { this.marketL2Args.push({ marketIndex: market.marketIndex, marketType: MarketType.SPOT, From 6a999959cc24b23c89a0fe9fbe2611f1b91a6eb4 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Fri, 27 Oct 2023 16:05:47 -0700 Subject: [PATCH 18/38] dlob subscriberio only uses market index --- src/dlob-subscriber/DLOBSubscriberIO.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index d371e75..64de205 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -80,7 +80,8 @@ export class DLOBSubscriberIO extends DLOBSubscriber { getL2AndSendMsg(l2Args: wsMarketL2Args): void { const grouping = l2Args.grouping; - const l2 = this.getL2(l2Args); + const {marketName, ...l2FuncArgs} = l2Args; + const l2 = this.getL2(l2FuncArgs); let l2Formatted: any; if (grouping) { const groupingBN = new BN(grouping); @@ -100,7 +101,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { this.lastSeenL2Formatted .get(l2Args.marketType) ?.set(l2Args.marketIndex, JSON.stringify(l2Formatted)); - l2Formatted['marketName'] = l2Args.marketName; + l2Formatted['marketName'] = marketName; l2Formatted['marketType'] = l2Args.marketType; l2Formatted['marketIndex'] = l2Args.marketIndex; l2Formatted['oracle'] = getOracleForMarket( @@ -109,7 +110,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { l2Args.marketIndex ); this.redisClient.client.publish( - l2Args.marketName, + marketName, JSON.stringify(l2Formatted) ); } From d5f1584bef55383a9fc956e4e3aef219768121aa Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Fri, 27 Oct 2023 16:06:05 -0700 Subject: [PATCH 19/38] prettify --- src/dlob-subscriber/DLOBSubscriberIO.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 64de205..c4fe2f6 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -80,7 +80,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { getL2AndSendMsg(l2Args: wsMarketL2Args): void { const grouping = l2Args.grouping; - const {marketName, ...l2FuncArgs} = l2Args; + const { marketName, ...l2FuncArgs } = l2Args; const l2 = this.getL2(l2FuncArgs); let l2Formatted: any; if (grouping) { @@ -109,9 +109,6 @@ export class DLOBSubscriberIO extends DLOBSubscriber { l2Args.marketType, l2Args.marketIndex ); - this.redisClient.client.publish( - marketName, - JSON.stringify(l2Formatted) - ); + this.redisClient.client.publish(marketName, JSON.stringify(l2Formatted)); } } From cc8fe46ad2ac54b691aeb0fa73fbb419bfe5a46c Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Fri, 27 Oct 2023 17:09:19 -0700 Subject: [PATCH 20/38] ws manager uses password --- package.json | 2 +- src/wsConnectionManager.ts | 13 ++++++++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 535cd2f..2458e73 100644 --- a/package.json +++ b/package.json @@ -50,7 +50,7 @@ "start": "node lib/index.js", "dev": "ts-node src/index.ts", "ws-publish": "ts-node src/wsPublish.ts", - "manager": "ts-node src/wsConnectionManager.ts", + "ws-manager": "ts-node src/wsConnectionManager.ts", "dev:inspect": "yarn build && node --inspect ./lib/index.js", "dev:debug": "yarn build && node --inspect-brk --inspect=2230 ./lib/index.js", "example": "ts-node example/client.ts", diff --git a/src/wsConnectionManager.ts b/src/wsConnectionManager.ts index 1faf185..efa8afa 100644 --- a/src/wsConnectionManager.ts +++ b/src/wsConnectionManager.ts @@ -16,9 +16,10 @@ const io = new Server(server); const REDIS_HOST = process.env.REDIS_HOST || 'localhost'; const REDIS_PORT = process.env.REDIS_PORT || '6379'; +const REDIS_PASSWORD = process.env.REDIS_PASSWORD; async function main() { - const redisClient = new RedisClient(REDIS_HOST, REDIS_PORT); + const redisClient = new RedisClient(REDIS_HOST, REDIS_PORT, REDIS_PASSWORD); await redisClient.connect(); io.on('connection', (socket) => { @@ -31,6 +32,16 @@ async function main() { } }); }); + + socket.on('unsubscribe', (channel) => { + console.log('Unsubscribing from channel', channel); + redisClient.client.unsubscribe(channel); + }); + + socket.on('disconnect', () => { + console.log('Client disconnected'); + }); + }); server.listen('3000', () => { From a130bac2cfb1806c95fdc55875c1d6d9a54833e9 Mon Sep 17 00:00:00 2001 From: wphan Date: Fri, 27 Oct 2023 22:29:09 -0700 Subject: [PATCH 21/38] wsConnectionManager listen on /ws --- src/wsConnectionManager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wsConnectionManager.ts b/src/wsConnectionManager.ts index efa8afa..de8eb1e 100644 --- a/src/wsConnectionManager.ts +++ b/src/wsConnectionManager.ts @@ -12,7 +12,7 @@ app.use(compression()); app.set('trust proxy', 1); const server = http.createServer(app); -const io = new Server(server); +const io = new Server(server, { path: '/ws' }); const REDIS_HOST = process.env.REDIS_HOST || 'localhost'; const REDIS_PORT = process.env.REDIS_PORT || '6379'; From 659bd65b3866ed353ffe10ff2150f3f2d68bea5e Mon Sep 17 00:00:00 2001 From: wphan Date: Fri, 27 Oct 2023 22:43:16 -0700 Subject: [PATCH 22/38] error handling --- src/wsConnectionManager.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/wsConnectionManager.ts b/src/wsConnectionManager.ts index de8eb1e..96d4814 100644 --- a/src/wsConnectionManager.ts +++ b/src/wsConnectionManager.ts @@ -42,6 +42,9 @@ async function main() { console.log('Client disconnected'); }); + socket.on('error', (error) => { + console.error('Socket error:', error); + }); }); server.listen('3000', () => { From 6895b8cb1781f2cea4a16778a14e03b604f8ca95 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Sun, 29 Oct 2023 12:49:38 -0700 Subject: [PATCH 23/38] adding client test for k8s --- src/clientTest.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/clientTest.ts diff --git a/src/clientTest.ts b/src/clientTest.ts new file mode 100644 index 0000000..913549e --- /dev/null +++ b/src/clientTest.ts @@ -0,0 +1,17 @@ +import { io } from 'socket.io-client'; + +const main = async () => { + const socket = io('http://localhost:3000'); + + socket.on('connect', () => { + socket.emit('subscribe', 'SOL-PERP'); + }); + + socket.on('SOL-PERP', (data: any) => { + console.log(data); + }); +}; + +main().then(() => { + console.log('running'); +}); From 3a4aa5be6794a6f5ec7c70b655225fd9236b8884 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Sun, 29 Oct 2023 14:08:07 -0700 Subject: [PATCH 24/38] using ws instead of socket.io --- package.json | 6 +- src/clientTest.ts | 17 ----- src/wsConnectionManager.ts | 58 +++++++++++----- yarn.lock | 134 +++++-------------------------------- 4 files changed, 60 insertions(+), 155 deletions(-) delete mode 100644 src/clientTest.ts diff --git a/package.json b/package.json index 2458e73..3b9ec7e 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "@project-serum/serum": "^0.13.65", "@solana/web3.js": "^1.22.0", "@types/redis": "^4.0.11", + "@types/ws": "^8.5.8", "async-mutex": "^0.4.0", "commander": "^9.4.0", "compression": "^1.7.4", @@ -25,11 +26,10 @@ "morgan": "^1.10.0", "redis": "^4.6.10", "response-time": "^2.3.2", - "socket.io": "^4.7.2", - "socket.io-client": "^4.7.2", "socket.io-redis": "^6.1.1", "typescript": "4.5.4", - "winston": "^3.8.1" + "winston": "^3.8.1", + "ws": "^8.14.2" }, "devDependencies": { "@types/k6": "^0.45.0", diff --git a/src/clientTest.ts b/src/clientTest.ts deleted file mode 100644 index 913549e..0000000 --- a/src/clientTest.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { io } from 'socket.io-client'; - -const main = async () => { - const socket = io('http://localhost:3000'); - - socket.on('connect', () => { - socket.emit('subscribe', 'SOL-PERP'); - }); - - socket.on('SOL-PERP', (data: any) => { - console.log(data); - }); -}; - -main().then(() => { - console.log('running'); -}); diff --git a/src/wsConnectionManager.ts b/src/wsConnectionManager.ts index 96d4814..743c74b 100644 --- a/src/wsConnectionManager.ts +++ b/src/wsConnectionManager.ts @@ -1,8 +1,8 @@ import cors from 'cors'; import express from 'express'; import * as http from 'http'; -import { Server } from 'socket.io'; import compression from 'compression'; +import { WebSocket, WebSocketServer } from 'ws'; import { sleep } from './utils/utils'; import { RedisClient } from './utils/redisClient'; @@ -12,7 +12,7 @@ app.use(compression()); app.set('trust proxy', 1); const server = http.createServer(app); -const io = new Server(server, { path: '/ws' }); +const wss = new WebSocketServer({server, path: '/ws'}); const REDIS_HOST = process.env.REDIS_HOST || 'localhost'; const REDIS_PORT = process.env.REDIS_PORT || '6379'; @@ -22,27 +22,51 @@ async function main() { const redisClient = new RedisClient(REDIS_HOST, REDIS_PORT, REDIS_PASSWORD); await redisClient.connect(); - io.on('connection', (socket) => { - socket.on('subscribe', (channel) => { - console.log('Subscribing to channel', channel); - redisClient.client.subscribe(channel); - redisClient.client.on('message', (subscribedChannel, message) => { - if (subscribedChannel === channel) { - socket.emit(channel, JSON.parse(message)); + wss.on('connection', (ws: WebSocket) => { + console.log('Client connected'); + + ws.on('message', async (msg) => { + const parsedMessage = JSON.parse(msg.toString()); + switch (parsedMessage.type) { + case 'subscribe': { + const channel = parsedMessage.channel; + console.log('Subscribing to channel', channel); + redisClient.client.subscribe(channel); + redisClient.client.on('message', (subscribedChannel, message) => { + if (subscribedChannel === channel) { + ws.send(JSON.stringify({channel, data: message})); + } + }); + break; } + case 'unsubscribe': { + const channel = parsedMessage.channel; + console.log('Unsubscribing from channel', channel); + redisClient.client.unsubscribe(channel); + } + } + }); + + // Ping/pong connection timeout + let isAlive = true; + + ws.on('pong', () => { + isAlive = true; + }); + setInterval(() => { + wss.clients.forEach((ws: WebSocket) => { + if (isAlive === false) return ws.terminate(); + + isAlive = false; + ws.ping(); }); - }); + }, 30000); - socket.on('unsubscribe', (channel) => { - console.log('Unsubscribing from channel', channel); - redisClient.client.unsubscribe(channel); - }); - - socket.on('disconnect', () => { + ws.on('disconnect', () => { console.log('Client disconnected'); }); - socket.on('error', (error) => { + ws.on('error', (error) => { console.error('Socket error:', error); }); }); diff --git a/yarn.lock b/yarn.lock index 6973f25..e6793a4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -940,11 +940,6 @@ resolved "https://registry.yarnpkg.com/@sideway/pinpoint/-/pinpoint-2.0.0.tgz#cff8ffadc372ad29fd3f78277aeb29e632cc70df" integrity sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ== -"@socket.io/component-emitter@~3.1.0": - version "3.1.0" - resolved "https://registry.yarnpkg.com/@socket.io/component-emitter/-/component-emitter-3.1.0.tgz#96116f2a912e0c02817345b3c10751069920d553" - integrity sha512-+9jVqKhRSpsc591z5vX+X5Yyw+he/HCB4iQ/RYxw35CEPaY1gnsNE43nf9n9AaYjAQrTiI/mOwKUKdUs9vf7Xg== - "@solana/buffer-layout-utils@^0.2.0": version "0.2.0" resolved "https://registry.yarnpkg.com/@solana/buffer-layout-utils/-/buffer-layout-utils-0.2.0.tgz#b45a6cab3293a2eb7597cceb474f229889d875ca" @@ -1134,11 +1129,6 @@ resolved "https://registry.yarnpkg.com/@types/content-disposition/-/content-disposition-0.5.5.tgz#650820e95de346e1f84e30667d168c8fd25aa6e3" integrity sha512-v6LCdKfK6BwcqMo+wYW05rLS12S0ZO0Fl4w1h4aaZMD7bqT3gVUns6FvLJKGZHQmYn3SX55JWGpziwJRwVgutA== -"@types/cookie@^0.4.1": - version "0.4.1" - resolved "https://registry.yarnpkg.com/@types/cookie/-/cookie-0.4.1.tgz#bfd02c1f2224567676c1545199f87c3a861d878d" - integrity sha512-XW/Aa8APYr6jSVVA1y/DEIZX0/GMKLEVekNG727R8cs56ahETkRAy/3DR7+fJyh7oUgGwNQaRfXCun0+KbWY7Q== - "@types/cookies@*": version "0.7.7" resolved "https://registry.yarnpkg.com/@types/cookies/-/cookies-0.7.7.tgz#7a92453d1d16389c05a5301eef566f34946cfd81" @@ -1149,13 +1139,6 @@ "@types/keygrip" "*" "@types/node" "*" -"@types/cors@^2.8.12": - version "2.8.15" - resolved "https://registry.yarnpkg.com/@types/cors/-/cors-2.8.15.tgz#eb143aa2f8807ddd78e83cbff141bbedd91b60ee" - integrity sha512-n91JxbNLD8eQIuXDIChAN1tCKNWCEgpceU9b7ZMbFA+P+Q4yIeh80jizFLEvolRPc1ES0VdwFlGv+kJTSirogw== - dependencies: - "@types/node" "*" - "@types/express-serve-static-core@^4.17.18": version "4.17.31" resolved "https://registry.yarnpkg.com/@types/express-serve-static-core/-/express-serve-static-core-4.17.31.tgz#a1139efeab4e7323834bb0226e62ac019f474b2f" @@ -1336,13 +1319,6 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-18.11.9.tgz#02d013de7058cea16d36168ef2fc653464cfbad4" integrity sha512-CRpX21/kGdzjOpFsZSkcrXMGIBWMGNIHXXBVFSH+ggkftxg+XYP20TESbh+zFvFj3EQOl5byk0HTRn1IL6hbqg== -"@types/node@>=10.0.0": - version "20.8.8" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.8.tgz#adee050b422061ad5255fc38ff71b2bb96ea2a0e" - integrity sha512-YRsdVxq6OaLfmR9Hy816IMp33xOBjfyOgUd77ehqg96CFywxAPbDbXvAsuN2KVg2HOT8Eh6uAfU+l4WffwPVrQ== - dependencies: - undici-types "~5.25.1" - "@types/node@^12.12.54": version "12.20.55" resolved "https://registry.yarnpkg.com/@types/node/-/node-12.20.55.tgz#c329cbd434c42164f846b909bd6f85b5537f6240" @@ -1425,6 +1401,13 @@ dependencies: "@types/node" "*" +"@types/ws@^8.5.8": + version "8.5.8" + resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.5.8.tgz#13efec7bd439d0bdf2af93030804a94f163b1430" + integrity sha512-flUksGIQCnJd6sZ1l5dqCEG/ksaoAg/eUwiLAGTJQcfgvZJKF++Ta4bJA6A5aPSJmsr+xlseHn4KLgVlNnvPTg== + dependencies: + "@types/node" "*" + "@typescript-eslint/eslint-plugin@^4.28.0": version "4.33.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-4.33.0.tgz#c24dc7c8069c7706bc40d99f6fa87edcb2005276" @@ -1508,7 +1491,7 @@ abstract-logging@^2.0.0: resolved "https://registry.yarnpkg.com/abstract-logging/-/abstract-logging-2.0.1.tgz#6b0c371df212db7129b57d2e7fcf282b8bf1c839" integrity sha512-2BjRTZxTPvheOvGbBslFSYOUkr+SjPtOnrLP33f+VIWLzezQpZcqVg7ja3L4dBXmzzgwT+a029jRx5PCi3JuiA== -accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.8: +accepts@~1.3.5, accepts@~1.3.8: version "1.3.8" resolved "https://registry.yarnpkg.com/accepts/-/accepts-1.3.8.tgz#0bf0be125b67014adcb0b0921e62db7bffe16b2e" integrity sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw== @@ -1690,11 +1673,6 @@ base64-js@^1.3.1, base64-js@^1.5.1: resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.5.1.tgz#1b1b440160a5bf7ad40b650f095963481903930a" integrity sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA== -base64id@2.0.0, base64id@~2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/base64id/-/base64id-2.0.0.tgz#2770ac6bc47d312af97a8bf9a634342e0cd25cb6" - integrity sha512-lGe34o6EHj9y3Kts9R4ZYs/Gr+6N7MCaMlIFA3F1R2O5/m7K06AxfSeO5530PEERE6/WyEg3lsuyw4GHlPZHog== - basic-auth@~2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-2.0.1.tgz#b998279bf47ce38344b4f3cf916d4679bbf51e3a" @@ -1990,12 +1968,7 @@ cookie@0.5.0, cookie@^0.5.0: resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.5.0.tgz#d1f5d71adec6558c58f389987c366aa47e994f8b" integrity sha512-YZ3GUyn/o8gfKJlnlX7g7xq4gyO6OSuhGPKaaGssGB2qgDUS0gPgtTvoyZLTt9Ab6dC4hfc9dV5arkvc/OCmrw== -cookie@~0.4.1: - version "0.4.2" - resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.2.tgz#0e41f24de5ecf317947c82fc789e06a884824432" - integrity sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA== - -cors@^2.8.5, cors@~2.8.5: +cors@^2.8.5: version "2.8.5" resolved "https://registry.yarnpkg.com/cors/-/cors-2.8.5.tgz#eac11da51592dd86b9f06f6e7ac293b3df875d29" integrity sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g== @@ -2036,7 +2009,7 @@ debug@2.6.9: dependencies: ms "2.0.0" -debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.3, debug@^4.3.4, debug@~4.3.1, debug@~4.3.2: +debug@^4.0.0, debug@^4.0.1, debug@^4.1.0, debug@^4.1.1, debug@^4.3.1, debug@^4.3.3, debug@^4.3.4, debug@~4.3.1: version "4.3.4" resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== @@ -2160,38 +2133,6 @@ end-of-stream@^1.4.1: dependencies: once "^1.4.0" -engine.io-client@~6.5.2: - version "6.5.2" - resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-6.5.2.tgz#8709e22c291d4297ae80318d3c8baeae71f0e002" - integrity sha512-CQZqbrpEYnrpGqC07a9dJDz4gePZUgTPMU3NKJPSeQOyw27Tst4Pl3FemKoFGAlHzgZmKjoRmiJvbWfhCXUlIg== - dependencies: - "@socket.io/component-emitter" "~3.1.0" - debug "~4.3.1" - engine.io-parser "~5.2.1" - ws "~8.11.0" - xmlhttprequest-ssl "~2.0.0" - -engine.io-parser@~5.2.1: - version "5.2.1" - resolved "https://registry.yarnpkg.com/engine.io-parser/-/engine.io-parser-5.2.1.tgz#9f213c77512ff1a6cc0c7a86108a7ffceb16fcfb" - integrity sha512-9JktcM3u18nU9N2Lz3bWeBgxVgOKpw7yhRaoxQA3FUDZzzw+9WlA6p4G4u0RixNkg14fH7EfEc/RhpurtiROTQ== - -engine.io@~6.5.2: - version "6.5.3" - resolved "https://registry.yarnpkg.com/engine.io/-/engine.io-6.5.3.tgz#80b0692912cef3a417e1b7433301d6397bf0374b" - integrity sha512-IML/R4eG/pUS5w7OfcDE0jKrljWS9nwnEfsxWCIJF5eO6AHo6+Hlv+lQbdlAYsiJPHzUthLm1RUjnBzWOs45cw== - dependencies: - "@types/cookie" "^0.4.1" - "@types/cors" "^2.8.12" - "@types/node" ">=10.0.0" - accepts "~1.3.4" - base64id "2.0.0" - cookie "~0.4.1" - cors "~2.8.5" - debug "~4.3.1" - engine.io-parser "~5.2.1" - ws "~8.11.0" - enquirer@^2.3.5: version "2.3.6" resolved "https://registry.yarnpkg.com/enquirer/-/enquirer-2.3.6.tgz#2a7fe5dd634a1e4125a975ec994ff5456dc3734d" @@ -3781,31 +3722,6 @@ socket.io-adapter@~2.2.0: resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-2.2.0.tgz#43af9157c4609e74b8addc6867873ac7eb48fda2" integrity sha512-rG49L+FwaVEwuAdeBRq49M97YI3ElVabJPzvHT9S6a2CWhDKnjSFasvwAwSYPRhQzfn4NtDIbCaGYgOCOU/rlg== -socket.io-adapter@~2.5.2: - version "2.5.2" - resolved "https://registry.yarnpkg.com/socket.io-adapter/-/socket.io-adapter-2.5.2.tgz#5de9477c9182fdc171cd8c8364b9a8894ec75d12" - integrity sha512-87C3LO/NOMc+eMcpcxUBebGjkpMDkNBS9tf7KJqcDsmL936EChtVva71Dw2q4tQcuVC+hAUy4an2NO/sYXmwRA== - dependencies: - ws "~8.11.0" - -socket.io-client@^4.7.2: - version "4.7.2" - resolved "https://registry.yarnpkg.com/socket.io-client/-/socket.io-client-4.7.2.tgz#f2f13f68058bd4e40f94f2a1541f275157ff2c08" - integrity sha512-vtA0uD4ibrYD793SOIAwlo8cj6haOeMHrGvwPxJsxH7CeIksqJ+3Zc06RvWTIFgiSqx4A3sOnTXpfAEE2Zyz6w== - dependencies: - "@socket.io/component-emitter" "~3.1.0" - debug "~4.3.2" - engine.io-client "~6.5.2" - socket.io-parser "~4.2.4" - -socket.io-parser@~4.2.4: - version "4.2.4" - resolved "https://registry.yarnpkg.com/socket.io-parser/-/socket.io-parser-4.2.4.tgz#c806966cf7270601e47469ddeec30fbdfda44c83" - integrity sha512-/GbIKmo8ioc+NIWIhwdecY0ge+qVBSMdgxGygevmdHj24bsfgtCmcUUcQ5ZzcylGFHsN3k4HB4Cgkl96KVnuew== - dependencies: - "@socket.io/component-emitter" "~3.1.0" - debug "~4.3.1" - socket.io-redis@^6.1.1: version "6.1.1" resolved "https://registry.yarnpkg.com/socket.io-redis/-/socket.io-redis-6.1.1.tgz#2361029a6c0b25c602d1422e1beb41907fd0e8bf" @@ -3817,19 +3733,6 @@ socket.io-redis@^6.1.1: socket.io-adapter "~2.2.0" uid2 "0.0.3" -socket.io@^4.7.2: - version "4.7.2" - resolved "https://registry.yarnpkg.com/socket.io/-/socket.io-4.7.2.tgz#22557d76c3f3ca48f82e73d68b7add36a22df002" - integrity sha512-bvKVS29/I5fl2FGLNHuXlQaUH/BlzX1IN6S+NKLNZpBsPZIDH+90eQmCs2Railn4YUiww4SzUedJ6+uzwFnKLw== - dependencies: - accepts "~1.3.4" - base64id "~2.0.0" - cors "~2.8.5" - debug "~4.3.2" - engine.io "~6.5.2" - socket.io-adapter "~2.5.2" - socket.io-parser "~4.2.4" - sonic-boom@^1.0.2: version "1.4.1" resolved "https://registry.yarnpkg.com/sonic-boom/-/sonic-boom-1.4.1.tgz#d35d6a74076624f12e6f917ade7b9d75e918f53e" @@ -4091,11 +3994,6 @@ uid2@0.0.3: resolved "https://registry.yarnpkg.com/uid2/-/uid2-0.0.3.tgz#483126e11774df2f71b8b639dcd799c376162b82" integrity sha512-5gSP1liv10Gjp8cMEnFd6shzkL/D6W1uhXSFNCxDC+YI8+L8wkCYCbJ7n77Ezb4wE/xzMogecE+DtamEe9PZjg== -undici-types@~5.25.1: - version "5.25.3" - resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.25.3.tgz#e044115914c85f0bcbb229f346ab739f064998c3" - integrity sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA== - unpipe@1.0.0, unpipe@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/unpipe/-/unpipe-1.0.0.tgz#b2bf4ee8514aae6165b4817829d21b2ef49904ec" @@ -4229,16 +4127,16 @@ ws@^7.4.5: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== -ws@^8.5.0, ws@~8.11.0: +ws@^8.14.2: + version "8.14.2" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.14.2.tgz#6c249a806eb2db7a20d26d51e7709eab7b2e6c7f" + integrity sha512-wEBG1ftX4jcglPxgFCMJmZ2PLtSbJ2Peg6TmpJFTbe9GZYOQCDPdMYu/Tm0/bGZkw8paZnJY45J4K2PZrLYq8g== + +ws@^8.5.0: version "8.11.0" resolved "https://registry.yarnpkg.com/ws/-/ws-8.11.0.tgz#6a0d36b8edfd9f96d8b25683db2f8d7de6e8e143" integrity sha512-HPG3wQd9sNQoT9xHyNCXoDUa+Xw/VevmY9FoHyQ+g+rrMn4j6FB4np7Z0OhdTgjx6MgQLK7jwSy1YecU1+4Asg== -xmlhttprequest-ssl@~2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/xmlhttprequest-ssl/-/xmlhttprequest-ssl-2.0.0.tgz#91360c86b914e67f44dce769180027c0da618c67" - integrity sha512-QKxVRxiRACQcVuQEYFsI1hhkrMlrXHPegbbd1yn9UHOmRxY+si12nQYzri3vbzt8VdTTRviqcKxcyllFas5z2A== - xtend@^4.0.0: version "4.0.2" resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54" From 547b8e85b9780e749273906cf8f513114dd40ef3 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Sun, 29 Oct 2023 14:08:46 -0700 Subject: [PATCH 25/38] changed client example to not use socket.io --- example/newClient.ts | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/example/newClient.ts b/example/newClient.ts index 913549e..5dba733 100644 --- a/example/newClient.ts +++ b/example/newClient.ts @@ -1,17 +1,26 @@ -import { io } from 'socket.io-client'; +import WebSocket from 'ws'; +const ws = new WebSocket('ws://localhost:3000/ws'); -const main = async () => { - const socket = io('http://localhost:3000'); - - socket.on('connect', () => { - socket.emit('subscribe', 'SOL-PERP'); - }); - - socket.on('SOL-PERP', (data: any) => { - console.log(data); - }); -}; - -main().then(() => { - console.log('running'); +ws.on('open', () => { + console.log('Connected to the server'); + ws.send(JSON.stringify({ type: 'subscribe', channel: 'SOL-PERP' })); +}); + +ws.on('message', (data: WebSocket.Data) => { + try { + const message = JSON.parse(data.toString()); + if (message.channel === 'SOL-PERP') { + console.log('Received data:', message.data); + } + } catch (e) { + console.error('Invalid message:', data); + } +}); + +ws.on('close', () => { + console.log('Disconnected from the server'); +}); + +ws.on('error', (error: Error) => { + console.error('WebSocket error:', error); }); From ee5cd4e6a86bc2c3f4f456e505d4dbe009a04839 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Sun, 29 Oct 2023 15:52:06 -0700 Subject: [PATCH 26/38] * much more robust ws connection manager --- src/wsConnectionManager.ts | 79 ++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 21 deletions(-) diff --git a/src/wsConnectionManager.ts b/src/wsConnectionManager.ts index 743c74b..41cdc1a 100644 --- a/src/wsConnectionManager.ts +++ b/src/wsConnectionManager.ts @@ -22,46 +22,83 @@ async function main() { const redisClient = new RedisClient(REDIS_HOST, REDIS_PORT, REDIS_PASSWORD); await redisClient.connect(); + const channelSubscribers = new Map>(); + const subscribedChannels = new Set(); + + redisClient.client.on('message', (subscribedChannel, message) => { + const subscribers = channelSubscribers.get(subscribedChannel); + subscribers.forEach((ws) => { + ws.send(JSON.stringify({channel: subscribedChannel, data: message })); + }); + }); + wss.on('connection', (ws: WebSocket) => { console.log('Client connected'); ws.on('message', async (msg) => { const parsedMessage = JSON.parse(msg.toString()); - switch (parsedMessage.type) { + + switch (parsedMessage.type.toLowerCase()) { case 'subscribe': { const channel = parsedMessage.channel; - console.log('Subscribing to channel', channel); - redisClient.client.subscribe(channel); - redisClient.client.on('message', (subscribedChannel, message) => { - if (subscribedChannel === channel) { - ws.send(JSON.stringify({channel, data: message})); - } - }); + if (!subscribedChannels.has(channel)) { + console.log('Subscribing to channel', channel); + redisClient.client.subscribe(channel).then(() => { + subscribedChannels.add(channel); + }).catch(() => { + ws.send(JSON.stringify({channel, error: `Invalid channel: ${channel}`})); + return; + }); + } + + if (!channelSubscribers.get(channel)) { + const subscribers = new Set(); + channelSubscribers.set(channel, subscribers); + } + channelSubscribers.get(channel).add(ws); break; } case 'unsubscribe': { const channel = parsedMessage.channel; - console.log('Unsubscribing from channel', channel); - redisClient.client.unsubscribe(channel); + const subscribers = channelSubscribers.get(channel); + if (subscribers) { + channelSubscribers.get(channel).delete(ws); + } + break; } + default: + break; } }); // Ping/pong connection timeout + let pongTimeoutId; let isAlive = true; - - ws.on('pong', () => { - isAlive = true; - }); - setInterval(() => { - wss.clients.forEach((ws: WebSocket) => { - if (isAlive === false) return ws.terminate(); - - isAlive = false; - ws.ping(); - }); + const pingIntervalId = setInterval(() => { + isAlive = false; + pongTimeoutId = setTimeout(() => { + if (!isAlive) { + console.log('Disconnecting because of ping/pong timeout'); + ws.terminate(); + } + }, 5000); // 5 seconds to wait for a pong + ws.ping(); }, 30000); + // Listen for pong messages + ws.on('pong', () => { + console.log('poooooooong'); + isAlive = true; + clearTimeout(pongTimeoutId); + }); + + // Handle disconnection + ws.on('close', () => { + // Clear any existing intervals and timeouts + clearInterval(pingIntervalId); + clearTimeout(pongTimeoutId); + }); + ws.on('disconnect', () => { console.log('Client disconnected'); }); From f3d64ff988f4ea2bf56b480f048203f307e06be9 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Sun, 29 Oct 2023 15:55:44 -0700 Subject: [PATCH 27/38] *prettify + new example --- example/{newClient.ts => wsClient.ts} | 15 +++++++++---- src/wsConnectionManager.ts | 32 +++++++++++++++++---------- 2 files changed, 31 insertions(+), 16 deletions(-) rename example/{newClient.ts => wsClient.ts} (50%) diff --git a/example/newClient.ts b/example/wsClient.ts similarity index 50% rename from example/newClient.ts rename to example/wsClient.ts index 5dba733..efb101b 100644 --- a/example/newClient.ts +++ b/example/wsClient.ts @@ -1,17 +1,24 @@ import WebSocket from 'ws'; +// const ws = new WebSocket('wss://master.dlob.drift.trade/ws'); const ws = new WebSocket('ws://localhost:3000/ws'); +import { sleep } from '../src/utils/utils'; -ws.on('open', () => { +ws.on('open', async () => { console.log('Connected to the server'); ws.send(JSON.stringify({ type: 'subscribe', channel: 'SOL-PERP' })); + ws.send(JSON.stringify({ type: 'subscribe', channel: 'LINK-PERP' })); + ws.send(JSON.stringify({ type: 'subscribe', channel: 'INJ-PERP' })); + await sleep(5000); + + ws.send(JSON.stringify({ type: 'unsubscribe', channel: 'SOL-PERP' })); + console.log("####################"); }); ws.on('message', (data: WebSocket.Data) => { try { const message = JSON.parse(data.toString()); - if (message.channel === 'SOL-PERP') { - console.log('Received data:', message.data); - } + console.log(`Received data from market ${message.channel}`); + // book data is in message.data } catch (e) { console.error('Invalid message:', data); } diff --git a/src/wsConnectionManager.ts b/src/wsConnectionManager.ts index 41cdc1a..867a5fe 100644 --- a/src/wsConnectionManager.ts +++ b/src/wsConnectionManager.ts @@ -12,7 +12,7 @@ app.use(compression()); app.set('trust proxy', 1); const server = http.createServer(app); -const wss = new WebSocketServer({server, path: '/ws'}); +const wss = new WebSocketServer({ server, path: '/ws' }); const REDIS_HOST = process.env.REDIS_HOST || 'localhost'; const REDIS_PORT = process.env.REDIS_PORT || '6379'; @@ -28,7 +28,7 @@ async function main() { redisClient.client.on('message', (subscribedChannel, message) => { const subscribers = channelSubscribers.get(subscribedChannel); subscribers.forEach((ws) => { - ws.send(JSON.stringify({channel: subscribedChannel, data: message })); + ws.send(JSON.stringify({ channel: subscribedChannel, data: message })); }); }); @@ -43,14 +43,22 @@ async function main() { const channel = parsedMessage.channel; if (!subscribedChannels.has(channel)) { console.log('Subscribing to channel', channel); - redisClient.client.subscribe(channel).then(() => { - subscribedChannels.add(channel); - }).catch(() => { - ws.send(JSON.stringify({channel, error: `Invalid channel: ${channel}`})); - return; - }); - } - + redisClient.client + .subscribe(channel) + .then(() => { + subscribedChannels.add(channel); + }) + .catch(() => { + ws.send( + JSON.stringify({ + channel, + error: `Invalid channel: ${channel}`, + }) + ); + return; + }); + } + if (!channelSubscribers.get(channel)) { const subscribers = new Set(); channelSubscribers.set(channel, subscribers); @@ -65,7 +73,7 @@ async function main() { channelSubscribers.get(channel).delete(ws); } break; - } + } default: break; } @@ -81,7 +89,7 @@ async function main() { console.log('Disconnecting because of ping/pong timeout'); ws.terminate(); } - }, 5000); // 5 seconds to wait for a pong + }, 5000); // 5 seconds to wait for a pong ws.ping(); }, 30000); From 5a4b08dce4724acfacb39e13b66f326a393432a8 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Sun, 29 Oct 2023 15:56:10 -0700 Subject: [PATCH 28/38] example bug fix --- example/wsClient.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/example/wsClient.ts b/example/wsClient.ts index efb101b..57678d9 100644 --- a/example/wsClient.ts +++ b/example/wsClient.ts @@ -1,6 +1,5 @@ import WebSocket from 'ws'; -// const ws = new WebSocket('wss://master.dlob.drift.trade/ws'); -const ws = new WebSocket('ws://localhost:3000/ws'); +const ws = new WebSocket('wss://master.dlob.drift.trade/ws'); import { sleep } from '../src/utils/utils'; ws.on('open', async () => { From d36a2c92975c7379eca1768d54ab2a87d260bb17 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 13:41:33 +0000 Subject: [PATCH 29/38] Bumping sdk dependency to 2.43.0-beta.2 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 3b9ec7e..c3c7e61 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.43.0-beta.1", + "@drift-labs/sdk": "2.43.0-beta.2", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index e6793a4..fabc67f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.43.0-beta.1": - version "2.43.0-beta.1" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.1.tgz#b1cc01fa5f1bdba6c4597b783f75cc24cf74909e" - integrity sha512-Ij0+C2d3bhoLhAMNqy4omqe8tN+CSAAOVUrgVCBNFr9LHwmUFITJLt11vpj355FdJxH1frR7EW2s4M3lZdsrog== +"@drift-labs/sdk@2.43.0-beta.2": + version "2.43.0-beta.2" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.2.tgz#9d1f4969dbbe5ccf1a57bbe69141fc30c22dc4e2" + integrity sha512-Yg7MDJI4B0ZM8/E9SdYsQSwvifHGL2yoVbpSBMm3ZDiQ/Kerk+vmOp/xy+DvomUPtVB7PmvhrUzRC54xiEtgNQ== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From bded53ae635942eea4b2d78d036eb23cbd60f47e Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 23:16:32 +0000 Subject: [PATCH 30/38] Bumping sdk dependency to 2.43.0-beta.3 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index c3c7e61..54bbd6c 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.43.0-beta.2", + "@drift-labs/sdk": "2.43.0-beta.3", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index fabc67f..97018b7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.43.0-beta.2": - version "2.43.0-beta.2" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.2.tgz#9d1f4969dbbe5ccf1a57bbe69141fc30c22dc4e2" - integrity sha512-Yg7MDJI4B0ZM8/E9SdYsQSwvifHGL2yoVbpSBMm3ZDiQ/Kerk+vmOp/xy+DvomUPtVB7PmvhrUzRC54xiEtgNQ== +"@drift-labs/sdk@2.43.0-beta.3": + version "2.43.0-beta.3" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.3.tgz#4e7345ff6b9694b449f3196edfb68f13135ae1fa" + integrity sha512-E3tUPNeboEUXP87j59QoanINfSlk19wJm7ybhFTx5WcIih5E2QcNj4Si8OhdsP6jKEEAw/pmRmAxgG5YuXQ0vg== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 8ebe8c7a4eeefc3ffbccf09f2400fe702f7d8fd2 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 30 Oct 2023 23:43:48 +0000 Subject: [PATCH 31/38] Bumping sdk dependency to 2.43.0-beta.4 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 54bbd6c..66119ba 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.43.0-beta.3", + "@drift-labs/sdk": "2.43.0-beta.4", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 97018b7..2b7ae3f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.43.0-beta.3": - version "2.43.0-beta.3" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.3.tgz#4e7345ff6b9694b449f3196edfb68f13135ae1fa" - integrity sha512-E3tUPNeboEUXP87j59QoanINfSlk19wJm7ybhFTx5WcIih5E2QcNj4Si8OhdsP6jKEEAw/pmRmAxgG5YuXQ0vg== +"@drift-labs/sdk@2.43.0-beta.4": + version "2.43.0-beta.4" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.4.tgz#c01e5dc29fca8c98e501b279a1db86592706103f" + integrity sha512-kylGivJ2v+b+KOMYC7jiKNJ8I5AsDS/CSb4NQjZF1d63PpwjxYe11veyWoeMeEzrscTHhvjtY6G1SiijMeqWFA== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 2bfa1ed91fec90a1ec6c4bcddfba3265d957159d Mon Sep 17 00:00:00 2001 From: Luke Steyn Date: Tue, 31 Oct 2023 12:58:48 +0100 Subject: [PATCH 32/38] Added WS_PORT config to wsConnectionManager --- src/wsConnectionManager.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/wsConnectionManager.ts b/src/wsConnectionManager.ts index 867a5fe..eea8707 100644 --- a/src/wsConnectionManager.ts +++ b/src/wsConnectionManager.ts @@ -6,6 +6,9 @@ import { WebSocket, WebSocketServer } from 'ws'; import { sleep } from './utils/utils'; import { RedisClient } from './utils/redisClient'; +// Set up env constants +require('dotenv').config(); + const app = express(); app.use(cors({ origin: '*' })); app.use(compression()); @@ -16,6 +19,7 @@ const wss = new WebSocketServer({ server, path: '/ws' }); const REDIS_HOST = process.env.REDIS_HOST || 'localhost'; const REDIS_PORT = process.env.REDIS_PORT || '6379'; +const WS_PORT = process.env.WS_PORT || '3000'; const REDIS_PASSWORD = process.env.REDIS_PASSWORD; async function main() { @@ -116,8 +120,8 @@ async function main() { }); }); - server.listen('3000', () => { - console.log('connection manager running on 3000'); + server.listen(WS_PORT, () => { + console.log(`connection manager running on ${WS_PORT}`); }); } From 8c8a653d5591cf06a44cdcb02a7c315ed7ed688f Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 Nov 2023 04:49:22 +0000 Subject: [PATCH 33/38] Bumping sdk dependency to 2.43.0-beta.5 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 66119ba..76316cc 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.43.0-beta.4", + "@drift-labs/sdk": "2.43.0-beta.5", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 2b7ae3f..56274e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.43.0-beta.4": - version "2.43.0-beta.4" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.4.tgz#c01e5dc29fca8c98e501b279a1db86592706103f" - integrity sha512-kylGivJ2v+b+KOMYC7jiKNJ8I5AsDS/CSb4NQjZF1d63PpwjxYe11veyWoeMeEzrscTHhvjtY6G1SiijMeqWFA== +"@drift-labs/sdk@2.43.0-beta.5": + version "2.43.0-beta.5" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.5.tgz#3e7ba3c5b5fb52bdf6028e56fd466532cdea776d" + integrity sha512-gJvJbC53cYGRMK7GTfz+I+XoTmOgUyexWzlaARDI47E+oJGyZH/jaFriQjyyHPDp9XNcqkGSpVDwT8JZ3MSbVg== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 20ed020e8b6ff6d3389fdf443ecfb5aa21ddab33 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 Nov 2023 16:36:23 +0000 Subject: [PATCH 34/38] Bumping sdk dependency to 2.43.0-beta.6 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 76316cc..9b324b6 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.43.0-beta.5", + "@drift-labs/sdk": "2.43.0-beta.6", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 56274e0..e67a050 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.43.0-beta.5": - version "2.43.0-beta.5" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.5.tgz#3e7ba3c5b5fb52bdf6028e56fd466532cdea776d" - integrity sha512-gJvJbC53cYGRMK7GTfz+I+XoTmOgUyexWzlaARDI47E+oJGyZH/jaFriQjyyHPDp9XNcqkGSpVDwT8JZ3MSbVg== +"@drift-labs/sdk@2.43.0-beta.6": + version "2.43.0-beta.6" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.6.tgz#d94799d2e3ac66a30333afb9b9b8452b3ef6ff3e" + integrity sha512-Llgqm3tyeZabB9zAYIm0RPAe+ZAZ3q4Zuq29wksLzYf5LK7bT4SWjYGf4UohaTq3ht1DR1ZxdP7xPsveYAnNLA== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 45f2b73ae67e38f3a0f337c563e57f1c93b7125a Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 3 Nov 2023 17:56:15 +0000 Subject: [PATCH 35/38] Bumping sdk dependency to 2.43.0-beta.7 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 9b324b6..a928b11 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.43.0-beta.6", + "@drift-labs/sdk": "2.43.0-beta.7", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index e67a050..aa7cdfd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.43.0-beta.6": - version "2.43.0-beta.6" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.6.tgz#d94799d2e3ac66a30333afb9b9b8452b3ef6ff3e" - integrity sha512-Llgqm3tyeZabB9zAYIm0RPAe+ZAZ3q4Zuq29wksLzYf5LK7bT4SWjYGf4UohaTq3ht1DR1ZxdP7xPsveYAnNLA== +"@drift-labs/sdk@2.43.0-beta.7": + version "2.43.0-beta.7" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.7.tgz#b93b7f186d3588cdf85af081414abb028cf3c928" + integrity sha512-XZTC/twPQOyB6kT5TTnC7VO8avjwiSy8DKSWvcFKB9eB+o01qlZuNE6bhUeY+ZnPIyk5Rd6bLVKVPH1umw+G4Q== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 2a49af5dc47ae2997428fac850471d8bf96ec5cd Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 6 Nov 2023 17:19:39 +0000 Subject: [PATCH 36/38] Bumping sdk dependency to 2.43.0-beta.8 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index a928b11..eec0c2c 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.43.0-beta.7", + "@drift-labs/sdk": "2.43.0-beta.8", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index aa7cdfd..cfad1e0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.43.0-beta.7": - version "2.43.0-beta.7" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.7.tgz#b93b7f186d3588cdf85af081414abb028cf3c928" - integrity sha512-XZTC/twPQOyB6kT5TTnC7VO8avjwiSy8DKSWvcFKB9eB+o01qlZuNE6bhUeY+ZnPIyk5Rd6bLVKVPH1umw+G4Q== +"@drift-labs/sdk@2.43.0-beta.8": + version "2.43.0-beta.8" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.8.tgz#00b4f88bcb618455c7c88e927dd65f2f9885c6e0" + integrity sha512-oJOdiRjB9TtPgHAwwNYSe2cecJ1p9JZTQQIT00hCY2aRcvI3kZyegrz/P7bChLqY6J3arAvrgE8315oGvXft2w== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From fb1ea4b3794531fabc36f6f045eddf741fd13a53 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 08:49:08 +0000 Subject: [PATCH 37/38] Bumping sdk dependency to 2.43.0-beta.9 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index eec0c2c..5a629ec 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.43.0-beta.8", + "@drift-labs/sdk": "2.43.0-beta.9", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index cfad1e0..649d08e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.43.0-beta.8": - version "2.43.0-beta.8" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.8.tgz#00b4f88bcb618455c7c88e927dd65f2f9885c6e0" - integrity sha512-oJOdiRjB9TtPgHAwwNYSe2cecJ1p9JZTQQIT00hCY2aRcvI3kZyegrz/P7bChLqY6J3arAvrgE8315oGvXft2w== +"@drift-labs/sdk@2.43.0-beta.9": + version "2.43.0-beta.9" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.9.tgz#45a77e1016265ebf3ec394c445503213b5128612" + integrity sha512-eMlUPZgVtmW8KvM6ijSoc+LyykWVzm2YBgmsZCEJoCclvsdq8lNlu++uWpcT0cPWUahDZj663EzZnKL2BCnQuA== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 21a09df9ed1c19362006a9b6457fc9ec9e4897af Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 7 Nov 2023 11:40:25 +0000 Subject: [PATCH 38/38] Bumping sdk dependency to 2.43.0-beta.10 --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 5a629ec..ecd18ff 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.43.0-beta.9", + "@drift-labs/sdk": "2.43.0-beta.10", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 649d08e..9ff7218 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.43.0-beta.9": - version "2.43.0-beta.9" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.9.tgz#45a77e1016265ebf3ec394c445503213b5128612" - integrity sha512-eMlUPZgVtmW8KvM6ijSoc+LyykWVzm2YBgmsZCEJoCclvsdq8lNlu++uWpcT0cPWUahDZj663EzZnKL2BCnQuA== +"@drift-labs/sdk@2.43.0-beta.10": + version "2.43.0-beta.10" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.10.tgz#afe05e7b6aaf557d495a2e4743767be4b6c27435" + integrity sha512-NqQ2jV7ewiAkL2nNdxnoi1C+tefnxskFo9rHtl6yUF6NpygE08usOcbCu5to+Cyrtx6cSAymHsxCrDU1c7lkQw== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2"