From 855ecbeed1e692f2c2a9de39d2fd7fac0da60e73 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 17:15:56 +0000 Subject: [PATCH 01/42] Bumping drift-common to 0b3b35b66fda60565b716ed5dfc5ccd5bd1a8d12 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 55975ac..0b3b35b 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 55975ac61a2369c201b3b8cbe046b9a101de75e3 +Subproject commit 0b3b35b66fda60565b716ed5dfc5ccd5bd1a8d12 From 8299cfc8493953e2f6b09a2e2c213a598912b709 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 28 Jul 2025 22:31:29 +0000 Subject: [PATCH 02/42] Bumping drift-common to 125dddf4237719e65b30b050f71a4b33120699c8 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 0b3b35b..125dddf 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 0b3b35b66fda60565b716ed5dfc5ccd5bd1a8d12 +Subproject commit 125dddf4237719e65b30b050f71a4b33120699c8 From 0f0054fdf22db409da0391c85fafc9ea646b162d Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 29 Jul 2025 14:11:14 +0000 Subject: [PATCH 03/42] Bumping drift-common to c749ccb8eb12145c3b5b0a44d57d7f03263640ac --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 125dddf..c749ccb 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 125dddf4237719e65b30b050f71a4b33120699c8 +Subproject commit c749ccb8eb12145c3b5b0a44d57d7f03263640ac From 4572c347695cdf2b5c53d023708e5f17d7d63cbd Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 29 Jul 2025 15:06:03 +0000 Subject: [PATCH 04/42] Bumping drift-common to 3083d43726213cf9d926a04bc513d225a6ea5401 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index c749ccb..3083d43 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit c749ccb8eb12145c3b5b0a44d57d7f03263640ac +Subproject commit 3083d43726213cf9d926a04bc513d225a6ea5401 From a64cc332885127c2f89c619e96e334a483385410 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 29 Jul 2025 15:39:30 +0000 Subject: [PATCH 05/42] Bumping drift-common to 1d52d5121b4a4063f0f297397b01a4f17791cc2e --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 3083d43..1d52d51 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 3083d43726213cf9d926a04bc513d225a6ea5401 +Subproject commit 1d52d5121b4a4063f0f297397b01a4f17791cc2e From ea7da6e3582df8926a160a1a70be5fd1a9a0b9ff Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 29 Jul 2025 17:49:18 +0000 Subject: [PATCH 06/42] Bumping drift-common to 134f81b4035ff65f4644162d0a0d2dfcf741fd54 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 1d52d51..134f81b 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 1d52d5121b4a4063f0f297397b01a4f17791cc2e +Subproject commit 134f81b4035ff65f4644162d0a0d2dfcf741fd54 From ff4a79c5b41639888166bed87f6fb5ad5edad103 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 29 Jul 2025 18:20:03 +0000 Subject: [PATCH 07/42] Bumping drift-common to 15121b9244bada02fe95bd02d82ea9556955944c --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 134f81b..15121b9 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 134f81b4035ff65f4644162d0a0d2dfcf741fd54 +Subproject commit 15121b9244bada02fe95bd02d82ea9556955944c From bb476e97f74374e5b536c4e5bc68c6c2dc87f9d8 Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Tue, 29 Jul 2025 11:30:11 -0700 Subject: [PATCH 08/42] use mm oracle --- build_all.sh | 4 +++ src/dlob-subscriber/DLOBSubscriberIO.ts | 8 +++--- src/utils/utils.ts | 34 ++++++++++++------------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/build_all.sh b/build_all.sh index 5333e79..40e9a9f 100644 --- a/build_all.sh +++ b/build_all.sh @@ -1,6 +1,10 @@ git submodule init git submodule update --recursive +rm -rf node_modules +rm -rf drift-common/protocol/sdk/node_modules +rm -rf drift-common/common-ts/node_modules + echo "building sdk..." cd drift-common/protocol/sdk yarn clean && yarn && yarn build && yarn link diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 20c2867..fe72fd3 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -13,7 +13,7 @@ import { PerpOperation, PositionDirection, ZERO, - calculateBidAskPrice, + calculateAMMBidAskPrice, getLimitPrice, isOperationPaused, isVariant, @@ -177,9 +177,11 @@ export class DLOBSubscriberIO extends DLOBSubscriber { let bestBid = bestDlobBid; let bestAsk = bestDlobAsk; if (marketType === 'perp') { - const [bestVammBid, bestVammAsk] = calculateBidAskPrice( + const [bestVammBid, bestVammAsk] = calculateAMMBidAskPrice( this.driftClient.getPerpMarketAccount(marketArgs.marketIndex).amm, - oraclePriceData, + this.driftClient.getMMOracleDataForPerpMarket( + marketArgs.marketIndex + ), true ); diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 351e140..c70ac70 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -1,7 +1,7 @@ import { BN, BigNum, - DriftClient, + IDriftClient, DriftEnv, L2OrderBook, L3OrderBook, @@ -92,7 +92,7 @@ export function parsePositiveIntArray( } export const getOracleForMarket = ( - driftClient: DriftClient, + driftClient: IDriftClient, marketType: MarketType, marketIndex: number ): number => { @@ -129,7 +129,7 @@ const getSerializableOraclePriceData = ( }; export const getOracleDataForMarket = ( - driftClient: DriftClient, + driftClient: IDriftClient, marketType: MarketType, marketIndex: number ): SerializableOraclePriceData => { @@ -146,7 +146,7 @@ export const getOracleDataForMarket = ( export const addOracletoResponse = ( response: L2OrderBook | L3OrderBook, - driftClient: DriftClient, + driftClient: IDriftClient, marketType: MarketType, marketIndex: number ): void => { @@ -177,7 +177,7 @@ export const addOracletoResponse = ( export const addMarketSlotToResponse = ( response: L2OrderBook | L3OrderBook, - driftClient: DriftClient, + driftClient: IDriftClient, marketType: MarketType, marketIndex: number ): void => { @@ -418,7 +418,7 @@ export const validateWsSubscribeMsg = ( }; export const validateDlobQuery = ( - driftClient: DriftClient, + driftClient: IDriftClient, driftEnv: DriftEnv, marketType?: string, marketIndex?: string, @@ -571,7 +571,7 @@ export function errorHandler( */ export const getPhoenixSubscriber = ( - driftClient: DriftClient, + driftClient: IDriftClient, marketConfig: SpotMarketConfig, sdkConfig ): PhoenixSubscriber => { @@ -586,7 +586,7 @@ export const getPhoenixSubscriber = ( }; export const getSerumSubscriber = ( - driftClient: DriftClient, + driftClient: IDriftClient, marketConfig: SpotMarketConfig, sdkConfig ): SerumSubscriber => { @@ -601,7 +601,7 @@ export const getSerumSubscriber = ( }; export const getOpenbookSubscriber = ( - driftClient: DriftClient, + driftClient: IDriftClient, marketConfig: SpotMarketConfig, sdkConfig ): OpenbookV2Subscriber => { @@ -748,7 +748,7 @@ export const convertRawL2ToBN = (rawL2: any): L2OrderBook => { /** * Get L2 orderbook data and calculate estimated prices - * @param driftClient - DriftClient instance + * @param driftClient - IDriftClient instance * @param marketType - MarketType enum * @param marketIndex - Market index number * @param direction - Position direction @@ -759,7 +759,7 @@ export const convertRawL2ToBN = (rawL2: any): L2OrderBook => { * @returns Price data object with oracle, best, entry, worst, and mark prices */ export const getEstimatedPrices = async ( - driftClient: DriftClient, + driftClient: IDriftClient, marketType: MarketType, marketIndex: number, direction: PositionDirection, @@ -857,14 +857,14 @@ export const getEstimatedPrices = async ( /** * Maps AuctionParamArgs to the format expected by deriveMarketOrderParams * @param params - AuctionParamArgs from the API request - * @param driftClient - DriftClient instance (optional, for price calculation) + * @param driftClient - IDriftClient instance (optional, for price calculation) * @param fetchFromRedis - Redis fetch function (optional, for price calculation) * @param selectMostRecentBySlot - Slot selection function (optional, for price calculation) * @returns Object formatted for deriveMarketOrderParams function or error response */ export const mapToMarketOrderParams = async ( params: AuctionParamArgs, - driftClient?: DriftClient, + driftClient?: IDriftClient, fetchFromRedis?: ( key: string, selectionCriteria: (responses: any) => any @@ -1064,14 +1064,14 @@ export const fetchL2FromRedis = async ( * @param direction - Position direction ('long' or 'short') * @param marketIndex - Market index number * @param marketType - Market type ('spot' or 'perp') - * @param driftClient - DriftClient instance for oracle data + * @param driftClient - IDriftClient instance for oracle data * @param l2Formatted - Already formatted L2OrderBook data * @returns Dynamic slippage tolerance as a number */ export const calculateDynamicSlippage = ( marketIndex: number, marketType: string, - driftClient: DriftClient, + driftClient: IDriftClient, l2Formatted: L2OrderBook ): number => { // Determine if this is a major market (PERP with marketIndex 0, 1, or 2) @@ -1128,7 +1128,7 @@ export const calculateDynamicSlippage = ( /** * Get L2 orderbook data and calculate estimated prices using pre-fetched L2 data - * @param driftClient - DriftClient instance + * @param driftClient - IDriftClient instance * @param marketType - MarketType enum * @param marketIndex - Market index number * @param direction - Position direction @@ -1138,7 +1138,7 @@ export const calculateDynamicSlippage = ( * @returns Price data object with oracle, best, entry, worst, and mark prices */ export const getEstimatedPricesWithL2 = async ( - driftClient: DriftClient, + driftClient: IDriftClient, marketType: MarketType, marketIndex: number, direction: PositionDirection, From 86ad4b6c278d8aaf9ddafc3f1fbfbf0be6e26458 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 29 Jul 2025 20:58:37 +0000 Subject: [PATCH 09/42] Bumping drift-common to 2eb8c75456334a5d757d1e085eddff0fec95ff4e --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 15121b9..2eb8c75 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 15121b9244bada02fe95bd02d82ea9556955944c +Subproject commit 2eb8c75456334a5d757d1e085eddff0fec95ff4e From f813c4e973783ccf19ca55897b1c6227b19b2b4f Mon Sep 17 00:00:00 2001 From: Nour Alharithi Date: Tue, 29 Jul 2025 15:56:51 -0700 Subject: [PATCH 10/42] remove IClient references --- src/utils/utils.ts | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/utils/utils.ts b/src/utils/utils.ts index c70ac70..351e140 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -1,7 +1,7 @@ import { BN, BigNum, - IDriftClient, + DriftClient, DriftEnv, L2OrderBook, L3OrderBook, @@ -92,7 +92,7 @@ export function parsePositiveIntArray( } export const getOracleForMarket = ( - driftClient: IDriftClient, + driftClient: DriftClient, marketType: MarketType, marketIndex: number ): number => { @@ -129,7 +129,7 @@ const getSerializableOraclePriceData = ( }; export const getOracleDataForMarket = ( - driftClient: IDriftClient, + driftClient: DriftClient, marketType: MarketType, marketIndex: number ): SerializableOraclePriceData => { @@ -146,7 +146,7 @@ export const getOracleDataForMarket = ( export const addOracletoResponse = ( response: L2OrderBook | L3OrderBook, - driftClient: IDriftClient, + driftClient: DriftClient, marketType: MarketType, marketIndex: number ): void => { @@ -177,7 +177,7 @@ export const addOracletoResponse = ( export const addMarketSlotToResponse = ( response: L2OrderBook | L3OrderBook, - driftClient: IDriftClient, + driftClient: DriftClient, marketType: MarketType, marketIndex: number ): void => { @@ -418,7 +418,7 @@ export const validateWsSubscribeMsg = ( }; export const validateDlobQuery = ( - driftClient: IDriftClient, + driftClient: DriftClient, driftEnv: DriftEnv, marketType?: string, marketIndex?: string, @@ -571,7 +571,7 @@ export function errorHandler( */ export const getPhoenixSubscriber = ( - driftClient: IDriftClient, + driftClient: DriftClient, marketConfig: SpotMarketConfig, sdkConfig ): PhoenixSubscriber => { @@ -586,7 +586,7 @@ export const getPhoenixSubscriber = ( }; export const getSerumSubscriber = ( - driftClient: IDriftClient, + driftClient: DriftClient, marketConfig: SpotMarketConfig, sdkConfig ): SerumSubscriber => { @@ -601,7 +601,7 @@ export const getSerumSubscriber = ( }; export const getOpenbookSubscriber = ( - driftClient: IDriftClient, + driftClient: DriftClient, marketConfig: SpotMarketConfig, sdkConfig ): OpenbookV2Subscriber => { @@ -748,7 +748,7 @@ export const convertRawL2ToBN = (rawL2: any): L2OrderBook => { /** * Get L2 orderbook data and calculate estimated prices - * @param driftClient - IDriftClient instance + * @param driftClient - DriftClient instance * @param marketType - MarketType enum * @param marketIndex - Market index number * @param direction - Position direction @@ -759,7 +759,7 @@ export const convertRawL2ToBN = (rawL2: any): L2OrderBook => { * @returns Price data object with oracle, best, entry, worst, and mark prices */ export const getEstimatedPrices = async ( - driftClient: IDriftClient, + driftClient: DriftClient, marketType: MarketType, marketIndex: number, direction: PositionDirection, @@ -857,14 +857,14 @@ export const getEstimatedPrices = async ( /** * Maps AuctionParamArgs to the format expected by deriveMarketOrderParams * @param params - AuctionParamArgs from the API request - * @param driftClient - IDriftClient instance (optional, for price calculation) + * @param driftClient - DriftClient instance (optional, for price calculation) * @param fetchFromRedis - Redis fetch function (optional, for price calculation) * @param selectMostRecentBySlot - Slot selection function (optional, for price calculation) * @returns Object formatted for deriveMarketOrderParams function or error response */ export const mapToMarketOrderParams = async ( params: AuctionParamArgs, - driftClient?: IDriftClient, + driftClient?: DriftClient, fetchFromRedis?: ( key: string, selectionCriteria: (responses: any) => any @@ -1064,14 +1064,14 @@ export const fetchL2FromRedis = async ( * @param direction - Position direction ('long' or 'short') * @param marketIndex - Market index number * @param marketType - Market type ('spot' or 'perp') - * @param driftClient - IDriftClient instance for oracle data + * @param driftClient - DriftClient instance for oracle data * @param l2Formatted - Already formatted L2OrderBook data * @returns Dynamic slippage tolerance as a number */ export const calculateDynamicSlippage = ( marketIndex: number, marketType: string, - driftClient: IDriftClient, + driftClient: DriftClient, l2Formatted: L2OrderBook ): number => { // Determine if this is a major market (PERP with marketIndex 0, 1, or 2) @@ -1128,7 +1128,7 @@ export const calculateDynamicSlippage = ( /** * Get L2 orderbook data and calculate estimated prices using pre-fetched L2 data - * @param driftClient - IDriftClient instance + * @param driftClient - DriftClient instance * @param marketType - MarketType enum * @param marketIndex - Market index number * @param direction - Position direction @@ -1138,7 +1138,7 @@ export const calculateDynamicSlippage = ( * @returns Price data object with oracle, best, entry, worst, and mark prices */ export const getEstimatedPricesWithL2 = async ( - driftClient: IDriftClient, + driftClient: DriftClient, marketType: MarketType, marketIndex: number, direction: PositionDirection, From c615e8d0ae97c8672a995fffceff1ab685eb3de2 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 29 Jul 2025 23:29:06 +0000 Subject: [PATCH 11/42] Bumping drift-common to 4e0782ca2077bd60373299dbb07fe1d554f20a99 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 2eb8c75..4e0782c 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 2eb8c75456334a5d757d1e085eddff0fec95ff4e +Subproject commit 4e0782ca2077bd60373299dbb07fe1d554f20a99 From af97a9b04ea8ba864a00565085087054311eddd4 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 30 Jul 2025 00:28:59 +0000 Subject: [PATCH 12/42] Bumping drift-common to e6d037888c6cdb8c8cfb9673a26f716a31ae15ae --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 4e0782c..e6d0378 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 4e0782ca2077bd60373299dbb07fe1d554f20a99 +Subproject commit e6d037888c6cdb8c8cfb9673a26f716a31ae15ae From 3c859ea38c2d746b42a5894f74ac379f41ff26a5 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 30 Jul 2025 01:18:29 +0000 Subject: [PATCH 13/42] Bumping drift-common to f457571598358278b95c29c59a8f224a800b02f1 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index e6d0378..f457571 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit e6d037888c6cdb8c8cfb9673a26f716a31ae15ae +Subproject commit f457571598358278b95c29c59a8f224a800b02f1 From a4225eca2a09652d8418c84629b9b4e8bb276cb7 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 30 Jul 2025 04:26:10 +0000 Subject: [PATCH 14/42] Bumping drift-common to 3a30d4105323128d954e17aa76cb89dd180a15b1 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index f457571..3a30d41 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit f457571598358278b95c29c59a8f224a800b02f1 +Subproject commit 3a30d4105323128d954e17aa76cb89dd180a15b1 From 82283c24f6738fb22ae88cf1dc4b7170654f12c2 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 30 Jul 2025 09:06:21 +0000 Subject: [PATCH 15/42] Bumping drift-common to b0ea94c6912658d6c362209691e49f9b7bac6bdb --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 3a30d41..b0ea94c 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 3a30d4105323128d954e17aa76cb89dd180a15b1 +Subproject commit b0ea94c6912658d6c362209691e49f9b7bac6bdb From 33d27cf1867a498a8b00092fe0a3bd1ae493c5c2 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 30 Jul 2025 09:14:27 +0000 Subject: [PATCH 16/42] Bumping drift-common to bcbf6516a9720355bfb32641292600f10051129b --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index b0ea94c..bcbf651 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit b0ea94c6912658d6c362209691e49f9b7bac6bdb +Subproject commit bcbf6516a9720355bfb32641292600f10051129b From fc1f2226786841e07687ffc3f2821faf90edc2b8 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 30 Jul 2025 15:51:26 +0000 Subject: [PATCH 17/42] Bumping drift-common to 3d39da6eca391e576b093b8e136fda955eacc256 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index bcbf651..3d39da6 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit bcbf6516a9720355bfb32641292600f10051129b +Subproject commit 3d39da6eca391e576b093b8e136fda955eacc256 From ea4334ea3ac57245778f229d922014927a5ee0ec Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 30 Jul 2025 17:01:50 +0000 Subject: [PATCH 18/42] Bumping drift-common to 89d3d23f7afe767421784fe3aa4095f0019a1a1f --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 3d39da6..89d3d23 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 3d39da6eca391e576b093b8e136fda955eacc256 +Subproject commit 89d3d23f7afe767421784fe3aa4095f0019a1a1f From f5de37fb39d275833951455b8e79faecb2d0f76c Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 30 Jul 2025 18:41:52 +0000 Subject: [PATCH 19/42] Bumping drift-common to 8f94c374ad4de9e81b2e8ca197945b1ab36e7a2c --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 89d3d23..8f94c37 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 89d3d23f7afe767421784fe3aa4095f0019a1a1f +Subproject commit 8f94c374ad4de9e81b2e8ca197945b1ab36e7a2c From a1421764b3148e8a59ebaad2c3924b771b4aee42 Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Tue, 29 Jul 2025 13:05:21 -0600 Subject: [PATCH 20/42] feat: bump drift common --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 8f94c37..b2751c8 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 8f94c374ad4de9e81b2e8ca197945b1ab36e7a2c +Subproject commit b2751c84a158e82420ac3709292cd9b4dd485687 From 3179eea5f0b983e9b52733707ae52c4cd9116910 Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Tue, 29 Jul 2025 15:39:02 -0600 Subject: [PATCH 21/42] feat: use new ws v2 subscriber --- src/publishers/dlobPublisher.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index aead36e..7416f71 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -9,7 +9,6 @@ import { UserMap, Wallet, BulkAccountLoader, - OrderSubscriber, SlotSource, DriftClientSubscriptionConfig, SlotSubscriber, @@ -18,10 +17,9 @@ import { PerpMarketConfig, SpotMarketConfig, PhoenixSubscriber, - MarketType, - OraclePriceData, - ONE, decodeName, + WebSocketAccountSubscriberV2, + ONE, } from '@drift-labs/sdk'; import { RedisClient, RedisClientPrefix } from '@drift/common/clients'; @@ -29,7 +27,6 @@ import { logger, setLogLevel } from '../utils/logger'; import { SubscriberLookup, getOpenbookSubscriber, - l2WithBNToStrings, parsePositiveIntArray, sleep, } from '../utils/utils'; @@ -391,6 +388,7 @@ const main = async () => { commitment: stateCommitment, resubTimeoutMs: 30_000, logResubMessages: true, + accountSubscriber: WebSocketAccountSubscriberV2, }; slotSubscriber = new SlotSubscriber(connection, { resubTimeoutMs: 10_000, From b037606ecb4986f7c560f7fb86cccb71351ece2c Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Wed, 30 Jul 2025 14:17:49 -0600 Subject: [PATCH 22/42] chore: bump common --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index b2751c8..8f94c37 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit b2751c84a158e82420ac3709292cd9b4dd485687 +Subproject commit 8f94c374ad4de9e81b2e8ca197945b1ab36e7a2c From f234ad208aefbe8294b28ceee1ec799eb3666cb6 Mon Sep 17 00:00:00 2001 From: Nick Caradonna Date: Mon, 28 Jul 2025 17:58:16 -0400 Subject: [PATCH 23/42] speed up dlob publisher messages --- src/index.ts | 2 +- src/publishers/dlobPublisher.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/index.ts b/src/index.ts index fe6382f..cf635a2 100644 --- a/src/index.ts +++ b/src/index.ts @@ -79,7 +79,7 @@ const sdkConfig = initialize({ env: process.env.ENV }); const stateCommitment: Commitment = 'confirmed'; const serverPort = process.env.PORT || 6969; -export const ORDERBOOK_UPDATE_INTERVAL = 1000; +export const ORDERBOOK_UPDATE_INTERVAL = 500; const WS_FALLBACK_FETCH_INTERVAL = ORDERBOOK_UPDATE_INTERVAL * 60; const useWebsocket = process.env.USE_WEBSOCKET?.toLowerCase() === 'true'; const hermesUrl = process.env.HERMES_ENDPOINT; diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 7416f71..89c9242 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -110,7 +110,7 @@ const useOrderSubscriber = process.env.USE_ORDER_SUBSCRIBER?.toLowerCase() === 'true'; const ORDERBOOK_UPDATE_INTERVAL = - parseInt(process.env.ORDERBOOK_UPDATE_INTERVAL) || 1000; + parseInt(process.env.ORDERBOOK_UPDATE_INTERVAL) || 500; const WS_FALLBACK_FETCH_INTERVAL = 60_000; const KILLSWITCH_SLOT_DIFF_THRESHOLD = From 3f04ab072340fb1d336e12fdd428e19704900473 Mon Sep 17 00:00:00 2001 From: Nick Caradonna Date: Mon, 28 Jul 2025 18:22:46 -0400 Subject: [PATCH 24/42] lower to once per slot and lower long tail slippage --- src/index.ts | 2 +- src/publishers/dlobPublisher.ts | 2 +- src/utils/utils.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/index.ts b/src/index.ts index cf635a2..55b1305 100644 --- a/src/index.ts +++ b/src/index.ts @@ -79,7 +79,7 @@ const sdkConfig = initialize({ env: process.env.ENV }); const stateCommitment: Commitment = 'confirmed'; const serverPort = process.env.PORT || 6969; -export const ORDERBOOK_UPDATE_INTERVAL = 500; +export const ORDERBOOK_UPDATE_INTERVAL = 400; const WS_FALLBACK_FETCH_INTERVAL = ORDERBOOK_UPDATE_INTERVAL * 60; const useWebsocket = process.env.USE_WEBSOCKET?.toLowerCase() === 'true'; const hermesUrl = process.env.HERMES_ENDPOINT; diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 89c9242..1cf5328 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -110,7 +110,7 @@ const useOrderSubscriber = process.env.USE_ORDER_SUBSCRIBER?.toLowerCase() === 'true'; const ORDERBOOK_UPDATE_INTERVAL = - parseInt(process.env.ORDERBOOK_UPDATE_INTERVAL) || 500; + parseInt(process.env.ORDERBOOK_UPDATE_INTERVAL) || 400; const WS_FALLBACK_FETCH_INTERVAL = 60_000; const KILLSWITCH_SLOT_DIFF_THRESHOLD = diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 351e140..6c6aaba 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -1116,7 +1116,7 @@ export const calculateDynamicSlippage = ( // Apply multiplier from env var const multiplier = isMajor ? parseFloat(process.env.DYNAMIC_SLIPPAGE_MULTIPLIER_MAJOR || '1.02') - : parseFloat(process.env.DYNAMIC_SLIPPAGE_MULTIPLIER_NON_MAJOR || '1.5'); + : parseFloat(process.env.DYNAMIC_SLIPPAGE_MULTIPLIER_NON_MAJOR || '1.4'); dynamicSlippage = dynamicSlippage * multiplier; // Enforce minimum and maximum limits from env vars From 9f27a7db90082bdeb73cd01828a80ce6ea9b2707 Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Wed, 30 Jul 2025 22:48:25 -0600 Subject: [PATCH 25/42] fix: perp market acct subscriber prop wrong name drift client --- src/publishers/dlobPublisher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 1cf5328..d24f121 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -388,7 +388,7 @@ const main = async () => { commitment: stateCommitment, resubTimeoutMs: 30_000, logResubMessages: true, - accountSubscriber: WebSocketAccountSubscriberV2, + perpMarketAccountSubscriber: WebSocketAccountSubscriberV2, }; slotSubscriber = new SlotSubscriber(connection, { resubTimeoutMs: 10_000, From ba5fba3d80b7c6172e6bbe800580b2a2141ba342 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 31 Jul 2025 14:54:59 +0000 Subject: [PATCH 26/42] Bumping drift-common to 78fb5eb65d6b7a57846a37d2974654fadedb1186 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 8f94c37..78fb5eb 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 8f94c374ad4de9e81b2e8ca197945b1ab36e7a2c +Subproject commit 78fb5eb65d6b7a57846a37d2974654fadedb1186 From 5f683c849e80ff1a1f43a6506d0e566c347daa84 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 31 Jul 2025 15:00:44 +0000 Subject: [PATCH 27/42] Bumping drift-common to f2add490b80755a66fb66e0600c741e61ac277e9 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 78fb5eb..f2add49 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 78fb5eb65d6b7a57846a37d2974654fadedb1186 +Subproject commit f2add490b80755a66fb66e0600c741e61ac277e9 From 2ea2be9033e2d1f12bbba394ac318301e6db7c8b Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 31 Jul 2025 15:15:36 +0000 Subject: [PATCH 28/42] Bumping drift-common to 77ebbc393deeb11606db8fcc7d67d930bdea80b7 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index f2add49..77ebbc3 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit f2add490b80755a66fb66e0600c741e61ac277e9 +Subproject commit 77ebbc393deeb11606db8fcc7d67d930bdea80b7 From 2bdcf8f8dd7d7a6cc6a6c0d9c3ddabcec6c2d035 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 31 Jul 2025 15:29:55 +0000 Subject: [PATCH 29/42] Bumping drift-common to 052c69b18b04595f0fd88f6778f0e2128d690754 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 77ebbc3..052c69b 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 77ebbc393deeb11606db8fcc7d67d930bdea80b7 +Subproject commit 052c69b18b04595f0fd88f6778f0e2128d690754 From d01841d950c31b38bb7a6ff9fcb017447aaef46d Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 31 Jul 2025 16:26:51 +0000 Subject: [PATCH 30/42] Bumping drift-common to 75a0dcbc083a83a5c38be37b432a5710f9046423 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 052c69b..75a0dcb 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 052c69b18b04595f0fd88f6778f0e2128d690754 +Subproject commit 75a0dcbc083a83a5c38be37b432a5710f9046423 From 5d4651e1a18b922030cea3e8ef7c2ed670984fcb Mon Sep 17 00:00:00 2001 From: Nick Caradonna Date: Thu, 31 Jul 2025 17:48:29 -0400 Subject: [PATCH 31/42] use env var for both orderbook update intervals --- src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index 55b1305..916c46c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -79,7 +79,7 @@ const sdkConfig = initialize({ env: process.env.ENV }); const stateCommitment: Commitment = 'confirmed'; const serverPort = process.env.PORT || 6969; -export const ORDERBOOK_UPDATE_INTERVAL = 400; +export const ORDERBOOK_UPDATE_INTERVAL = parseInt(process.env.ORDERBOOK_UPDATE_INTERVAL) || 400; const WS_FALLBACK_FETCH_INTERVAL = ORDERBOOK_UPDATE_INTERVAL * 60; const useWebsocket = process.env.USE_WEBSOCKET?.toLowerCase() === 'true'; const hermesUrl = process.env.HERMES_ENDPOINT; From edb05abe098d42a95003134b7227a4cb683b6a62 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 31 Jul 2025 21:58:05 +0000 Subject: [PATCH 32/42] Bumping drift-common to a58e5c3d46597b1c31034fa54c83f12549d9cdcb --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 75a0dcb..a58e5c3 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 75a0dcbc083a83a5c38be37b432a5710f9046423 +Subproject commit a58e5c3d46597b1c31034fa54c83f12549d9cdcb From 18a32753346fddb8a47279c97521fb4ebb56ddd3 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 31 Jul 2025 22:48:18 +0000 Subject: [PATCH 33/42] Bumping drift-common to 0cfa40f0994b5a0053f469e9eda1b531b01cc2eb --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index a58e5c3..0cfa40f 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit a58e5c3d46597b1c31034fa54c83f12549d9cdcb +Subproject commit 0cfa40f0994b5a0053f469e9eda1b531b01cc2eb From 587a2d6d9e8865ce5472224af9fe1d85cf4b0bad Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 31 Jul 2025 23:03:44 +0000 Subject: [PATCH 34/42] Bumping drift-common to 67214545014ef420d80b5d6fed8e11741311c54c --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 0cfa40f..6721454 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 0cfa40f0994b5a0053f469e9eda1b531b01cc2eb +Subproject commit 67214545014ef420d80b5d6fed8e11741311c54c From 9fb5652c674deaaaecbf8addda9d48c997b35b98 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 1 Aug 2025 00:34:37 +0000 Subject: [PATCH 35/42] Bumping drift-common to e1bfa3118b3b50e1379c3760416b6e225ed5ca3d --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 6721454..e1bfa31 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 67214545014ef420d80b5d6fed8e11741311c54c +Subproject commit e1bfa3118b3b50e1379c3760416b6e225ed5ca3d From 9df6c84abdbcf90b7989d336019a725cc70a388e Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Fri, 1 Aug 2025 14:55:53 -0600 Subject: [PATCH 36/42] feat: TOB monitoring logic --- .husky/pre-commit | 22 ++++ README.md | 115 +++++++++++++++----- scripts/check-secrets.sh | 145 +++++++++++++++++++++++++ src/publishers/dlobPublisher.ts | 186 +++++++++++++++++++++++++++++++- 4 files changed, 441 insertions(+), 27 deletions(-) create mode 100755 .husky/pre-commit create mode 100755 scripts/check-secrets.sh diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100755 index 0000000..fe7ea7b --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1,22 @@ +#!/usr/bin/env sh +. "$(dirname -- "$0")/_/husky.sh" + +# Pre-commit hook to check for potential secrets +echo "🔍 Running secret check..." + +# Run the secret checking script +if [ -f "scripts/check-secrets.sh" ]; then + ./scripts/check-secrets.sh + if [ $? -ne 0 ]; then + echo "❌ Pre-commit hook failed. Please fix the issues above before committing." + echo "💡 If these are false positives, you can:" + echo " 1. Add the file to .gitignore" + echo " 2. Use placeholder values instead of real secrets" + echo " 3. Bypass the hook with: git commit --no-verify" + exit 1 + fi +else + echo "⚠️ Warning: check-secrets.sh not found" +fi + +echo "✅ Pre-commit checks passed" \ No newline at end of file diff --git a/README.md b/README.md index aeb0aac..4066f58 100644 --- a/README.md +++ b/README.md @@ -14,51 +14,82 @@ This is the backend server that provides a REST API for the drift [DLOB](https://docs.drift.trade/about-v2/decentralized-orderbook). +## Features + +- Real-time DLOB data publishing +- Support for both perp and spot markets +- Multiple subscription methods (WebSocket, gRPC, polling) +- Health checks and metrics +- **TOB (Top of Book) monitoring for stuck orders** (see [TOB Monitoring Configuration](#tob-monitoring-configuration)) + # Run the server ## Setup The build dependencies + ``` git submodule update --init bash build_all.sh ``` First set the necessary environment variables: + ``` cp .env.example .env ``` -## Environment Configuration +## Security + +### Pre-commit Hook + +This repository uses [Husky](https://typicode.github.io/husky/) to manage Git hooks. A pre-commit hook automatically checks for potential secrets before each commit. The hook will: + +- Scan for RPC URLs, API keys, tokens, and other potential secrets +- Prevent commits that contain suspicious patterns +- Provide helpful guidance if false positives are detected + +The hook is automatically installed when you run `npm install` (via the `prepare` script). If you need to bypass it for a specific commit, use: + +```bash +git commit --no-verify +``` + +### Scripts + +The `scripts/` directory contains utility scripts: + +- `check-secrets.sh` - Secret detection script used by the pre-commit hook + +## Environment Variables To properly configure the DLOB server, set the following environment variables in your `.env` file: -| Variable | Description | Example Value | -|---------------------------|-----------------------------------------------------------|--------------------------------------------------------------------| -| `ENDPOINT` | The Solana RPC node http endpoint. | `https://your-private-rpc-node.com` | -| `WS_ENDPOINT` | The Solana RPC node websocket endpoint. | `wss://your-private-rpc-node.com` | -| `USE_WEBSOCKET` | Flag to enable WebSocket connection. | `true` | -| `USE_ORDER_SUBSCRIBER` | Flag to enable order subscriber DLOB source. | `true` | -| `DISABLE_GPA_REFRESH` | Flag to disable periodic refresh using `getProgramAccounts`. | `true` | -| `ENV` | The network environment the server is connecting to. | `mainnet-beta` | -| `PORT` | The port number the HTTP server listens on. | `6969` | -| `METRICS_PORT` | The port number for Prometheus metrics. | `9465` | -| `PRIVATE_KEY` | Path to the Solana private key file. | `/path/to/keypair.json` | -| `RATE_LIMIT_CALLS_PER_SECOND` | Maximum number of API calls per second. | `100` | -| `PERP_MARKETS_TO_LOAD` | Number of perpetual markets to load at startup. | `0` | -| `SPOT_MARKETS_TO_LOAD` | Number of spot markets to load at startup. | `5` | -| `ELASTICACHE_HOST` | (for websocket server) Redis host endpoint. | `localhost` | -| `ELASTICACHE_PORT` | (for websocket server) Redis port. | `6379` | -| `REDIS_CLIENT` | (for websocket server) Redis client type (DLOB/DLOB_HELIUS).| `DLOB` | -| `WS_PORT` | (for websocket server) The port to run the websocket server on.| `3000` | - +| Variable | Description | Example Value | +| ----------------------------- | --------------------------------------------------------------- | ----------------------------------- | +| `ENDPOINT` | The Solana RPC node http endpoint. | `https://your-private-rpc-node.com` | +| `WS_ENDPOINT` | The Solana RPC node websocket endpoint. | `wss://your-private-rpc-node.com` | +| `USE_WEBSOCKET` | Flag to enable WebSocket connection. | `true` | +| `USE_ORDER_SUBSCRIBER` | Flag to enable order subscriber DLOB source. | `true` | +| `DISABLE_GPA_REFRESH` | Flag to disable periodic refresh using `getProgramAccounts`. | `true` | +| `ENV` | The network environment the server is connecting to. | `mainnet-beta` | +| `PORT` | The port number the HTTP server listens on. | `6969` | +| `METRICS_PORT` | The port number for Prometheus metrics. | `9465` | +| `PRIVATE_KEY` | Path to the Solana private key file. | `/path/to/keypair.json` | +| `RATE_LIMIT_CALLS_PER_SECOND` | Maximum number of API calls per second. | `100` | +| `PERP_MARKETS_TO_LOAD` | Number of perpetual markets to load at startup. | `0` | +| `SPOT_MARKETS_TO_LOAD` | Number of spot markets to load at startup. | `5` | +| `ELASTICACHE_HOST` | (for websocket server) Redis host endpoint. | `localhost` | +| `ELASTICACHE_PORT` | (for websocket server) Redis port. | `6379` | +| `REDIS_CLIENT` | (for websocket server) Redis client type (DLOB/DLOB_HELIUS). | `DLOB` | +| `WS_PORT` | (for websocket server) The port to run the websocket server on. | `3000` | Note: multiple Redis hosts can be provided by providing a comma separated string. - ## HTTP mode The HTTP server as documented [here](https://drift-labs.github.io/v2-teacher/?python#orderbook-trades-dlob-server) can be run with, and by default accessible on `http://127.0.0.1:6969`: + ``` yarn run dev ``` @@ -68,22 +99,26 @@ yarn run dev The websocket server has 2 components, the `dlob-publisher` that takes frequent snapshots of the DLOB and publishes them to Redis, and `ws-manager` listens for new connections and sends the latest DLOB to ws clients, the two components communicate through Redis pub-sub. To run the websocket server, a Redis cache is required, and the following environment variables must be set: -* `REDIS_HOSTS` -* `REDIS_PASSWORDS` -* `REDIS_PORTS` + +- `REDIS_HOSTS` +- `REDIS_PASSWORDS` +- `REDIS_PORTS` In the first terminal, start the redis cluster: + ``` bash redisCluster.sh start bash redisCluster.sh create ``` In second terminal, run: + ``` yarn run dlob-publish ``` In a third terminal, run: + ``` yarn run ws-manager ``` @@ -91,6 +126,7 @@ yarn run ws-manager Then connect to the ws server at ws://127.0.0.1:3000 When you're done, stop the redis cluster: + ``` bash redisCluster.sh stop ``` @@ -99,4 +135,33 @@ bash redisCluster.sh stop Documentation for connecting to the dlob server are available [here](https://drift-labs.github.io/v2-teacher/?python#orderbook-trades-dlob-server) -TODO: complete client examples. \ No newline at end of file +TODO: complete client examples. + +## TOB (Top of Book) Monitoring [#tob-monitoring] + +The server includes a TOB monitoring feature that detects when order books become stuck due to ghost/stuck orders. This is particularly useful for gRPC connections that may miss updates. + +### Configuration + +Set the following environment variables to enable and configure TOB monitoring: + +- `ENABLE_TOB_MONITORING=true` - Enable TOB monitoring (default: false) +- `TOB_CHECK_INTERVAL=60000` - How often to check TOB (default: 60 seconds) +- `TOB_STUCK_THRESHOLD=60000` - How long TOB can be stuck before resubscribing (default: 60 seconds) +- `TOP_MONITORING_ENABLED_PERP_MARKETS=0,1,2` - Comma-separated list of perp market indexes to monitor for TOB (default: 0,1,2 for SOL-PERP, BTC-PERP, ETH-PERP) + +### How it works + +1. Checks if the current node is configured to handle any TOB monitoring markets +2. Only enables monitoring if the node has TOB monitoring markets configured +3. Monitors the top bid/ask prices for TOB monitoring perp markets on this node +4. If TOB hasn't changed for the configured threshold time, triggers a resubscribe +5. Performs unsubscribe → subscribe → fetch sequence on the OrderSubscriber instance +6. Logs warnings and updates metrics for monitoring + +### Metrics + +The following metrics are available for TOB monitoring: + +- `tob_resubscribe` - Counter for resubscribe attempts (with success/failure labels) +- `tob_stuck_duration` - Gauge showing how long TOB has been stuck for each market diff --git a/scripts/check-secrets.sh b/scripts/check-secrets.sh new file mode 100755 index 0000000..7386ce6 --- /dev/null +++ b/scripts/check-secrets.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +# Pre-commit hook to check for potential secrets +# This script checks for common patterns that might indicate secrets being committed + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo "🔍 Checking for potential secrets in staged files..." + +# Get list of staged files +STAGED_FILES=$(git diff --cached --name-only --diff-filter=ACM) + +if [ -z "$STAGED_FILES" ]; then + echo "✅ No files staged for commit" + exit 0 +fi + +# Patterns to check for (add more as needed) +PATTERNS=( + # RPC URLs and endpoints + "https://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + "wss://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + "ws://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + "http://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + + # API keys and tokens (common patterns) + "[a-zA-Z0-9]{32,}" # Long alphanumeric strings (likely tokens) + "sk-[a-zA-Z0-9]{20,}" # Stripe-like keys + "pk_[a-zA-Z0-9]{20,}" # Public keys + "sk_[a-zA-Z0-9]{20,}" # Secret keys + + # Private keys and addresses + "[1-9A-HJ-NP-Za-km-z]{32,}" # Base58 encoded strings (likely private keys) + "0x[a-fA-F0-9]{40}" # Ethereum addresses + "[1-9A-HJ-NP-Za-km-z]{44}" # Solana addresses + + # Common secret patterns + "password.*=.*['\"][^'\"]{8,}['\"]" + "secret.*=.*['\"][^'\"]{8,}['\"]" + "token.*=.*['\"][^'\"]{8,}['\"]" + "key.*=.*['\"][^'\"]{8,}['\"]" + "api_key.*=.*['\"][^'\"]{8,}['\"]" + "private_key.*=.*['\"][^'\"]{8,}['\"]" + + # Environment variables that might contain secrets + "ENDPOINT.*=.*['\"][^'\"]{10,}['\"]" + "WS_ENDPOINT.*=.*['\"][^'\"]{10,}['\"]" + "GRPC_ENDPOINT.*=.*['\"][^'\"]{10,}['\"]" + "TOKEN.*=.*['\"][^'\"]{8,}['\"]" + "API_KEY.*=.*['\"][^'\"]{8,}['\"]" + "SECRET.*=.*['\"][^'\"]{8,}['\"]" +) + +# Files to exclude from checking +EXCLUDE_PATTERNS=( + "node_modules" + ".git" + "*.lock" + "*.lockb" + "*.so" + "*.dll" + "*.exe" + "*.bin" + "*.min.js" + "*.min.css" + "*.map" + "*.log" + "*.tmp" + "*.cache" + "build/" + "dist/" + ".env.example" + "scripts/check-secrets.sh" # Exclude this script itself +) + +# Function to check if file should be excluded +should_exclude() { + local file="$1" + for pattern in "${EXCLUDE_PATTERNS[@]}"; do + if [[ "$file" == *"$pattern"* ]] || [[ "$file" =~ $pattern ]]; then + return 0 + fi + done + return 1 +} + +# Function to check for patterns in a file +check_file() { + local file="$1" + local found_secrets=false + + # Skip binary files + if file "$file" | grep -q "binary"; then + return 0 + fi + + # Check each pattern + for pattern in "${PATTERNS[@]}"; do + if git grep --cached "$pattern" "$file" >/dev/null 2>&1; then + echo -e "${RED}⚠️ Potential secret found in $file${NC}" + echo -e "${YELLOW}Pattern: $pattern${NC}" + git grep --cached "$pattern" "$file" | head -3 + echo "" + found_secrets=true + fi + done + + if [ "$found_secrets" = true ]; then + return 1 + fi + + return 0 +} + +# Check each staged file +has_secrets=false +for file in $STAGED_FILES; do + if should_exclude "$file"; then + echo -e "${GREEN}✅ Skipping excluded file: $file${NC}" + continue + fi + + if ! check_file "$file"; then + has_secrets=true + fi +done + +if [ "$has_secrets" = true ]; then + echo -e "${RED}❌ Potential secrets detected!${NC}" + echo -e "${YELLOW}Please review the above matches and ensure no real secrets are being committed.${NC}" + echo -e "${YELLOW}If these are false positives, you can:${NC}" + echo -e "${YELLOW}1. Add the file to .gitignore${NC}" + echo -e "${YELLOW}2. Use placeholder values instead of real secrets${NC}" + echo -e "${YELLOW}3. Add the pattern to the exclude list in this script${NC}" + exit 1 +else + echo -e "${GREEN}✅ No potential secrets detected${NC}" + exit 0 +fi \ No newline at end of file diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index d24f121..118ac40 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -87,6 +87,14 @@ const kinesisRecordsSentCounter = metricsV2.addCounter( 'kinesis_records_sent', 'Number of records sent to Kinesis' ); +const tobResubscribeCounter = metricsV2.addCounter( + 'tob_resubscribe', + 'Number of TOB resubscribes triggered' +); +const tobStuckGauge = metricsV2.addGauge( + 'tob_stuck_duration', + 'Duration TOB has been stuck for each market' +); metricsV2.finalizeObservables(); //@ts-ignore @@ -94,7 +102,7 @@ const sdkConfig = initialize({ env: process.env.ENV }); let driftClient: DriftClient; const opts = program.opts(); -setLogLevel(opts.debug ? 'debug' : 'info'); +setLogLevel('debug'); const useGrpc = process.env.USE_GRPC?.toLowerCase() === 'true'; const useWebsocket = process.env.USE_WEBSOCKET?.toLowerCase() === 'true'; @@ -116,6 +124,16 @@ const WS_FALLBACK_FETCH_INTERVAL = 60_000; const KILLSWITCH_SLOT_DIFF_THRESHOLD = parseInt(process.env.KILLSWITCH_SLOT_DIFF_THRESHOLD) || 200; +// TOB monitoring configuration +const ENABLE_TOB_MONITORING = + process.env.ENABLE_TOB_MONITORING?.toLowerCase() === 'true'; +const TOB_CHECK_INTERVAL = parseInt(process.env.TOB_CHECK_INTERVAL) || 60_000; // 1 minute +const TOB_STUCK_THRESHOLD = parseInt(process.env.TOB_STUCK_THRESHOLD) || 60_000; // 1 minute without change +const TOB_MONITORING_ENABLED_PERP_MARKETS = process.env + .TOB_MONITORING_ENABLED_PERP_MARKETS + ? parsePositiveIntArray(process.env.TOB_MONITORING_ENABLED_PERP_MARKETS) + : [0, 1, 2]; // Default to SOL-PERP, BTC-PERP, ETH-PERP + // comma separated list of perp market indexes to load: i.e. 0,1,2,3 const PERP_MARKETS_TO_LOAD = process.env.PERP_MARKETS_TO_LOAD !== undefined @@ -142,6 +160,16 @@ logger.info( ); logger.info(`DriftEnv: ${driftEnv}`); logger.info(`Commit: ${commitHash}`); +logger.info( + `TOB Monitoring: ${ENABLE_TOB_MONITORING ? 'enabled' : 'disabled'}` +); +if (ENABLE_TOB_MONITORING) { + logger.info(`TOB Check Interval: ${TOB_CHECK_INTERVAL}ms`); + logger.info(`TOB Stuck Threshold: ${TOB_STUCK_THRESHOLD}ms`); + logger.info( + `TOB Monitoring Markets: ${TOB_MONITORING_ENABLED_PERP_MARKETS.join(', ')}` + ); +} let MARKET_SUBSCRIBERS: SubscriberLookup = {}; @@ -437,6 +465,8 @@ const main = async () => { ); let dlobProvider: DLOBProvider; + let orderSubscriber: OrderSubscriberFiltered | undefined; + if (useOrderSubscriber) { let subscriptionConfig: any = { type: 'polling', @@ -480,7 +510,7 @@ const main = async () => { }; } - const orderSubscriber = new OrderSubscriberFiltered({ + orderSubscriber = new OrderSubscriberFiltered({ driftClient, subscriptionConfig, ignoreList, @@ -623,6 +653,158 @@ const main = async () => { }); }, 10_000); + // TOB monitoring for configured perp markets to detect stuck orders + // Check if this node has any TOB monitoring markets configured + const tobMonitoringMarketsInThisNode = + TOB_MONITORING_ENABLED_PERP_MARKETS.filter((marketIndex) => + perpMarketInfos.some((market) => market.marketIndex === marketIndex) + ); + + const shouldEnableTobMonitoring = + ENABLE_TOB_MONITORING && + useOrderSubscriber && + tobMonitoringMarketsInThisNode.length > 0; + + // Track last TOB update times and values for each TOB monitoring market + const lastTobUpdateTimes = new Map(); + const lastTobValues = new Map(); + + // Initialize TOB tracking for TOB monitoring markets in this node + tobMonitoringMarketsInThisNode.forEach((marketIndex) => { + lastTobUpdateTimes.set(marketIndex, Date.now()); + lastTobValues.set(marketIndex, { bid: '', ask: '' }); + }); + + // Log TOB monitoring status + if (ENABLE_TOB_MONITORING) { + logger.info( + `TOB Monitoring Markets in this node: ${tobMonitoringMarketsInThisNode.join( + ', ' + )}` + ); + logger.info( + `TOB Monitoring active: ${shouldEnableTobMonitoring ? 'yes' : 'no'}` + ); + } + + // TOB monitoring function + const checkTobForStuckOrders = async () => { + if (!shouldEnableTobMonitoring) { + return; // Only monitor when using OrderSubscriber, TOB monitoring is enabled, and node has major markets + } + + logger.debug('Starting TOB monitoring check'); + + const currentTime = Date.now(); + + for (const marketIndex of tobMonitoringMarketsInThisNode) { + try { + // Get current TOB from DLOB + const slot = slotSource.getSlot(); + const dlob = await dlobProvider.getDLOB(slot); + + // Get oracle data for the market + const oracleData = driftClient.getOracleDataForPerpMarket(marketIndex); + + // Get L2 orderbook to check TOB + const l2OrderBook = dlob.getL2({ + marketIndex, + marketType: { perp: {} }, + slot, + oraclePriceData: oracleData, + depth: 1, // Only need top level + }); + + const bestBid = l2OrderBook.bids[0]?.price; + const bestAsk = l2OrderBook.asks[0]?.price; + + if (bestBid && bestAsk) { + const currentTobValues = { + bid: bestBid.toString(), + ask: bestAsk.toString(), + }; + const lastTobValue = lastTobValues.get(marketIndex); + const lastUpdate = lastTobUpdateTimes.get(marketIndex); + + // Check if TOB has changed + const tobChanged = + !lastTobValue || + lastTobValue.bid !== currentTobValues.bid || + lastTobValue.ask !== currentTobValues.ask; + + if (tobChanged) { + // TOB changed, update tracking + lastTobValues.set(marketIndex, currentTobValues); + lastTobUpdateTimes.set(marketIndex, currentTime); + logger.debug( + `TOB updated for market ${marketIndex}: bid=${currentTobValues.bid}, ask=${currentTobValues.ask}` + ); + } else if ( + lastUpdate && + currentTime - lastUpdate > TOB_STUCK_THRESHOLD + ) { + // TOB has been stuck for too long, trigger resubscribe + const stuckDuration = (currentTime - lastUpdate) / 1000; + logger.warn( + `TOB stuck for market ${marketIndex} for ${stuckDuration}s, triggering resubscribe` + ); + + // Update metrics + tobStuckGauge.setLatestValue(stuckDuration, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + }); + + // Get the OrderSubscriber instance for resubscribe + if (orderSubscriber) { + try { + // Resubscribe and fetch to clear stuck state + await orderSubscriber.unsubscribe(); + await orderSubscriber.subscribe(); + await orderSubscriber.fetch(); + + logger.info( + `Successfully resubscribed OrderSubscriber for market ${marketIndex}` + ); + + // Update metrics + tobResubscribeCounter.add(1, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + success: 'true', + }); + + // Reset the timer after successful resubscribe + lastTobUpdateTimes.set(marketIndex, currentTime); + } catch (error) { + logger.error( + `Failed to resubscribe OrderSubscriber for market ${marketIndex}:`, + error + ); + + // Update metrics for failed resubscribe + tobResubscribeCounter.add(1, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + success: 'false', + }); + } + } else { + logger.error( + `OrderSubscriber not available for market ${marketIndex}` + ); + } + } + } + } catch (error) { + logger.error(`Error checking TOB for market ${marketIndex}:`, error); + } + } + }; + + // Start TOB monitoring + setInterval(checkTobForStuckOrders, TOB_CHECK_INTERVAL); + const handleStartup = async (_req, res, _next) => { if (driftClient.isSubscribed && dlobProvider.size() > 0) { res.writeHead(200); From 6caec8e8e1a9b5aa996bc751f20ef8776b7e0118 Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Fri, 1 Aug 2025 16:00:23 -0600 Subject: [PATCH 37/42] refactor: TOB check uses user+orderId instead of prices --- .vscode/launch.json | 28 ++++++++++++++++++ src/publishers/dlobPublisher.ts | 50 ++++++++++++++++++++------------- 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 3cce174..2c2d82b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,6 +11,34 @@ "port": 2230, "timeout": 3000, "restart": true + }, + { + "type": "node", + "request": "launch", + "name": "Launch DLOB Publisher", + "runtimeExecutable": "ts-node", + "program": "${workspaceFolder}/src/publishers/dlobPublisher.ts", + "args": ["--D"], + "env": { + "KILLSWITCH_SLOT_DIFF_THRESHOLD": "2500", + "PERP_MARKETS_TO_LOAD": "1,2", + "LOCAL_CACHE": "true", + "RUNNING_LOCAL": "true", + "USE_REDIS": "true", + "ENDPOINT": "", + "WS_ENDPOINT": "", + "USE_WEBSOCKET": "true", + "USE_GRPC": "true", + "GRPC_ENDPOINT": "", + "TOKEN": "", + "USE_ORDER_SUBSCRIBER": "true", + "STATE_COMMITMENT": "confirmed", + "DRIFT_ENV": "mainnet-beta", + "ENV": "mainnet-beta", + "ENABLE_TOB_MONITORING": "true", + "TOB_MONITORING_ENABLED_PERP_MARKETS": "1,2" + }, + "console": "integratedTerminal" } ] } diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 118ac40..47a439a 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -665,14 +665,17 @@ const main = async () => { useOrderSubscriber && tobMonitoringMarketsInThisNode.length > 0; - // Track last TOB update times and values for each TOB monitoring market + // Track last TOB update times and order IDs for each TOB monitoring market const lastTobUpdateTimes = new Map(); - const lastTobValues = new Map(); + const lastTobOrderIds = new Map< + number, + { bidOrderId: string; askOrderId: string } + >(); // Initialize TOB tracking for TOB monitoring markets in this node tobMonitoringMarketsInThisNode.forEach((marketIndex) => { lastTobUpdateTimes.set(marketIndex, Date.now()); - lastTobValues.set(marketIndex, { bid: '', ask: '' }); + lastTobOrderIds.set(marketIndex, { bidOrderId: '', askOrderId: '' }); }); // Log TOB monitoring status @@ -706,38 +709,45 @@ const main = async () => { // Get oracle data for the market const oracleData = driftClient.getOracleDataForPerpMarket(marketIndex); - // Get L2 orderbook to check TOB - const l2OrderBook = dlob.getL2({ + // Get L3 orderbook to check TOB + const l3OrderBook = dlob.getL3({ marketIndex, marketType: { perp: {} }, slot, oraclePriceData: oracleData, - depth: 1, // Only need top level }); - const bestBid = l2OrderBook.bids[0]?.price; - const bestAsk = l2OrderBook.asks[0]?.price; + const bestBidOrder = l3OrderBook.bids[0]; + const bestAskOrder = l3OrderBook.asks[0]; - if (bestBid && bestAsk) { - const currentTobValues = { - bid: bestBid.toString(), - ask: bestAsk.toString(), + if (bestBidOrder && bestAskOrder) { + // Create order ID strings using maker (user account) and orderId + const currentBidOrderId = `${bestBidOrder.maker.toBase58()}-${ + bestBidOrder.orderId + }`; + const currentAskOrderId = `${bestAskOrder.maker.toBase58()}-${ + bestAskOrder.orderId + }`; + + const currentTobOrderIds = { + bidOrderId: currentBidOrderId, + askOrderId: currentAskOrderId, }; - const lastTobValue = lastTobValues.get(marketIndex); + const lastTobOrderId = lastTobOrderIds.get(marketIndex); const lastUpdate = lastTobUpdateTimes.get(marketIndex); - // Check if TOB has changed + // Check if TOB orders have changed (not just prices) const tobChanged = - !lastTobValue || - lastTobValue.bid !== currentTobValues.bid || - lastTobValue.ask !== currentTobValues.ask; + !lastTobOrderId || + lastTobOrderId.bidOrderId !== currentTobOrderIds.bidOrderId || + lastTobOrderId.askOrderId !== currentTobOrderIds.askOrderId; if (tobChanged) { - // TOB changed, update tracking - lastTobValues.set(marketIndex, currentTobValues); + // TOB orders changed, update tracking + lastTobOrderIds.set(marketIndex, currentTobOrderIds); lastTobUpdateTimes.set(marketIndex, currentTime); logger.debug( - `TOB updated for market ${marketIndex}: bid=${currentTobValues.bid}, ask=${currentTobValues.ask}` + `TOB orders updated for market ${marketIndex}: bidOrderId=${currentTobOrderIds.bidOrderId}, askOrderId=${currentTobOrderIds.askOrderId}` ); } else if ( lastUpdate && From 1c165dd11acc039ca125037373754ac64fa0e002 Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Sun, 3 Aug 2025 10:40:50 -0600 Subject: [PATCH 38/42] feat: handle case where TOB is empty for bids or asks for TOB checker --- src/publishers/dlobPublisher.ts | 152 ++++++++++++++++---------------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 47a439a..ffd81de 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -720,90 +720,90 @@ const main = async () => { const bestBidOrder = l3OrderBook.bids[0]; const bestAskOrder = l3OrderBook.asks[0]; - if (bestBidOrder && bestAskOrder) { - // Create order ID strings using maker (user account) and orderId - const currentBidOrderId = `${bestBidOrder.maker.toBase58()}-${ - bestBidOrder.orderId - }`; - const currentAskOrderId = `${bestAskOrder.maker.toBase58()}-${ - bestAskOrder.orderId - }`; + // Track each side independently, even if one side is empty + const currentBidOrderId = bestBidOrder + ? `${bestBidOrder.maker.toBase58()}-${bestBidOrder.orderId}` + : ''; + const currentAskOrderId = bestAskOrder + ? `${bestAskOrder.maker.toBase58()}-${bestAskOrder.orderId}` + : ''; - const currentTobOrderIds = { - bidOrderId: currentBidOrderId, - askOrderId: currentAskOrderId, - }; - const lastTobOrderId = lastTobOrderIds.get(marketIndex); - const lastUpdate = lastTobUpdateTimes.get(marketIndex); + const currentTobOrderIds = { + bidOrderId: currentBidOrderId, + askOrderId: currentAskOrderId, + }; + const lastTobOrderId = lastTobOrderIds.get(marketIndex); + const lastUpdate = lastTobUpdateTimes.get(marketIndex); - // Check if TOB orders have changed (not just prices) - const tobChanged = - !lastTobOrderId || - lastTobOrderId.bidOrderId !== currentTobOrderIds.bidOrderId || - lastTobOrderId.askOrderId !== currentTobOrderIds.askOrderId; + // Check if TOB orders have changed on either side + const tobChanged = + !lastTobOrderId || + lastTobOrderId.bidOrderId !== currentTobOrderIds.bidOrderId || + lastTobOrderId.askOrderId !== currentTobOrderIds.askOrderId; - if (tobChanged) { - // TOB orders changed, update tracking - lastTobOrderIds.set(marketIndex, currentTobOrderIds); - lastTobUpdateTimes.set(marketIndex, currentTime); - logger.debug( - `TOB orders updated for market ${marketIndex}: bidOrderId=${currentTobOrderIds.bidOrderId}, askOrderId=${currentTobOrderIds.askOrderId}` - ); - } else if ( - lastUpdate && - currentTime - lastUpdate > TOB_STUCK_THRESHOLD - ) { - // TOB has been stuck for too long, trigger resubscribe - const stuckDuration = (currentTime - lastUpdate) / 1000; - logger.warn( - `TOB stuck for market ${marketIndex} for ${stuckDuration}s, triggering resubscribe` - ); + if (tobChanged) { + // TOB orders changed, update tracking + lastTobOrderIds.set(marketIndex, currentTobOrderIds); + lastTobUpdateTimes.set(marketIndex, currentTime); + logger.debug( + `TOB orders updated for market ${marketIndex}: bidOrderId=${ + currentTobOrderIds.bidOrderId || 'none' + }, askOrderId=${currentTobOrderIds.askOrderId || 'none'}` + ); + } else if ( + lastUpdate && + currentTime - lastUpdate > TOB_STUCK_THRESHOLD + ) { + // TOB has been stuck for too long, trigger resubscribe + const stuckDuration = (currentTime - lastUpdate) / 1000; + logger.warn( + `TOB stuck for market ${marketIndex} for ${stuckDuration}s, triggering resubscribe` + ); - // Update metrics - tobStuckGauge.setLatestValue(stuckDuration, { - marketIndex: marketIndex.toString(), - marketType: 'perp', - }); + // Update metrics + tobStuckGauge.setLatestValue(stuckDuration, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + }); - // Get the OrderSubscriber instance for resubscribe - if (orderSubscriber) { - try { - // Resubscribe and fetch to clear stuck state - await orderSubscriber.unsubscribe(); - await orderSubscriber.subscribe(); - await orderSubscriber.fetch(); + // Get the OrderSubscriber instance for resubscribe + if (orderSubscriber) { + try { + // Resubscribe and fetch to clear stuck state + await orderSubscriber.unsubscribe(); + await orderSubscriber.subscribe(); + await orderSubscriber.fetch(); - logger.info( - `Successfully resubscribed OrderSubscriber for market ${marketIndex}` - ); - - // Update metrics - tobResubscribeCounter.add(1, { - marketIndex: marketIndex.toString(), - marketType: 'perp', - success: 'true', - }); - - // Reset the timer after successful resubscribe - lastTobUpdateTimes.set(marketIndex, currentTime); - } catch (error) { - logger.error( - `Failed to resubscribe OrderSubscriber for market ${marketIndex}:`, - error - ); - - // Update metrics for failed resubscribe - tobResubscribeCounter.add(1, { - marketIndex: marketIndex.toString(), - marketType: 'perp', - success: 'false', - }); - } - } else { - logger.error( - `OrderSubscriber not available for market ${marketIndex}` + logger.info( + `Successfully resubscribed OrderSubscriber for market ${marketIndex}` ); + + // Update metrics + tobResubscribeCounter.add(1, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + success: 'true', + }); + + // Reset the timer after successful resubscribe + lastTobUpdateTimes.set(marketIndex, currentTime); + } catch (error) { + logger.error( + `Failed to resubscribe OrderSubscriber for market ${marketIndex}:`, + error + ); + + // Update metrics for failed resubscribe + tobResubscribeCounter.add(1, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + success: 'false', + }); } + } else { + logger.error( + `OrderSubscriber not available for market ${marketIndex}` + ); } } } catch (error) { From fd2bf7a7b936835e016eac2fba3e10bf53c6e433 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 4 Aug 2025 08:58:32 +0000 Subject: [PATCH 39/42] Bumping drift-common to a3e15ea4e1d24705ec1e42d30c68c45c13828a6f --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index e1bfa31..a3e15ea 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit e1bfa3118b3b50e1379c3760416b6e225ed5ca3d +Subproject commit a3e15ea4e1d24705ec1e42d30c68c45c13828a6f From 24cfbefe16f779fabbefb3c216fce823a858ea72 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:16:46 +0000 Subject: [PATCH 40/42] Bumping drift-common to 3a4a4aba8d05c6fcff79f4fe868e2e702f6d54bf --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index a3e15ea..3a4a4ab 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit a3e15ea4e1d24705ec1e42d30c68c45c13828a6f +Subproject commit 3a4a4aba8d05c6fcff79f4fe868e2e702f6d54bf From 325eef5de3829fb8238a1c3f1d62937472131045 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 4 Aug 2025 15:58:27 +0000 Subject: [PATCH 41/42] Bumping drift-common to a9b2094da3f77715b1d41f2913496fcf03b2a97f --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 3a4a4ab..a9b2094 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 3a4a4aba8d05c6fcff79f4fe868e2e702f6d54bf +Subproject commit a9b2094da3f77715b1d41f2913496fcf03b2a97f From 862b2ff43b2a2c2be34175dab8df682b35182591 Mon Sep 17 00:00:00 2001 From: Nick Caradonna Date: Mon, 4 Aug 2025 14:02:34 -0400 Subject: [PATCH 42/42] make slippage tolerance in decimal form, fix dynamic slippage calc --- src/utils/utils.ts | 58 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 3 deletions(-) diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 6c6aaba..5fcb17f 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -25,6 +25,7 @@ import { PERCENTAGE_PRECISION_EXP, } from '@drift-labs/sdk'; import { RedisClient } from '@drift/common/clients'; +import { TradeOffsetPrice } from '@drift/common'; import { logger } from './logger'; import { NextFunction, Request, Response } from 'express'; import FEATURE_FLAGS from './featureFlags'; @@ -746,6 +747,37 @@ export const convertRawL2ToBN = (rawL2: any): L2OrderBook => { }; }; +/** + * Maps TradeOffsetPrice values to corresponding property names in estimatedPrices object + * @param offsetFrom - TradeOffsetPrice type or 'marketBased' or undefined + * @returns Property name string for accessing estimatedPrices + */ +export const mapTradeOffsetPriceToProperty = ( + offsetFrom: TradeOffsetPrice | 'marketBased' | undefined +): string => { + switch (offsetFrom) { + case 'best': + return 'bestPrice'; + case 'worst': + return 'worstPrice'; + case 'oracle': + return 'oraclePrice'; + case 'mark': + return 'markPrice'; + case 'entry': + return 'entryPrice'; + case 'bestOffer': + // For bestOffer, we'll use the best price (could be refined based on direction) + return 'bestPrice'; + case 'marketBased': + // Default to mark price for market-based pricing + return 'markPrice'; + default: + // Default fallback to mark price + return 'markPrice'; + } +}; + /** * Get L2 orderbook data and calculate estimated prices * @param driftClient - DriftClient instance @@ -935,11 +967,18 @@ export const mapToMarketOrderParams = async ( }; } + const startPriceProperty = mapTradeOffsetPriceToProperty( + params.auctionStartPriceOffsetFrom + ); + const startPrice = estimatedPrices[startPriceProperty]; + processedSlippageTolerance = calculateDynamicSlippage( params.marketIndex, params.marketType, driftClient, - l2Formatted + l2Formatted, + startPrice, + estimatedPrices.worstPrice ); } } else { @@ -1072,7 +1111,9 @@ export const calculateDynamicSlippage = ( marketIndex: number, marketType: string, driftClient: DriftClient, - l2Formatted: L2OrderBook + l2Formatted: L2OrderBook, + startPrice: BN, + worstPrice: BN ): number => { // Determine if this is a major market (PERP with marketIndex 0, 1, or 2) const isPerp = marketType.toLowerCase() === 'perp'; @@ -1113,6 +1154,17 @@ export const calculateDynamicSlippage = ( let dynamicSlippage = baseSlippage + spreadBaseSlippage; + // use halfway to worst price as size adjusted slippage + if (startPrice && worstPrice) { + const sizeAdjustedSlippage = + (startPrice.sub(worstPrice).abs().toNumber() / + BN.max(startPrice, worstPrice).toNumber() / + 2) * + 100; + + dynamicSlippage = Math.max(dynamicSlippage, sizeAdjustedSlippage); + } + // Apply multiplier from env var const multiplier = isMajor ? parseFloat(process.env.DYNAMIC_SLIPPAGE_MULTIPLIER_MAJOR || '1.02') @@ -1123,7 +1175,7 @@ export const calculateDynamicSlippage = ( const minSlippage = parseFloat(process.env.DYNAMIC_SLIPPAGE_MIN || '0.05'); // 0.05% minimum const maxSlippage = parseFloat(process.env.DYNAMIC_SLIPPAGE_MAX || '5'); // 5% maximum - return Math.min(Math.max(dynamicSlippage, minSlippage), maxSlippage); + return Math.min(Math.max(dynamicSlippage, minSlippage), maxSlippage) / 100; }; /**