Merge pull request #83 from drift-labs/master

market slot kill switch

package.json

@@ -6,7 +6,7 @@
   "license": "Apache-2.0",
   "dependencies": {
     "@coral-xyz/anchor": "^0.29.0",
-    "@drift-labs/sdk": "2.59.0-beta.0",
+    "@drift-labs/sdk": "2.59.0-beta.5",
     "@opentelemetry/api": "^1.1.0",
     "@opentelemetry/auto-instrumentations-node": "^0.31.1",
     "@opentelemetry/exporter-prometheus": "^0.31.0",

src/dlob-subscriber/DLOBSubscriberIO.ts

@@ -27,6 +27,8 @@ type wsMarketL2Args = {
 	updateOnChange?: boolean;
 };
 
+const MAKRET_STALENESS_THRESHOLD = 10 * 60 * 1000;
+
 export type wsMarketInfo = {
 	marketIndex: number;
 	marketName: string;
@@ -37,6 +39,10 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
 	public lastSeenL2Formatted: Map<MarketType, Map<number, any>>;
 	redisClient: RedisClient;
 	public killSwitchSlotDiffThreshold: number;
+	public lastMarketSlotMap: Map<
+		MarketType,
+		Map<number, { slot: number; ts: number }>
+	>;
 
 	constructor(
 		config: DLOBSubscriptionConfig & {
@@ -56,6 +62,9 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
 		this.lastSeenL2Formatted = new Map();
 		this.lastSeenL2Formatted.set(MarketType.SPOT, new Map());
 		this.lastSeenL2Formatted.set(MarketType.PERP, new Map());
+		this.lastMarketSlotMap = new Map();
+		this.lastMarketSlotMap.set(MarketType.SPOT, new Map());
+		this.lastMarketSlotMap.set(MarketType.PERP, new Map());
 
 		for (const market of config.perpMarketInfos) {
 			const perpMarket = this.driftClient.getPerpMarketAccount(
@@ -107,6 +116,15 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
 		const { marketName, ...l2FuncArgs } = l2Args;
 		const l2 = this.getL2(l2FuncArgs);
 		const slot = l2.slot;
+		const lastMarketSlotAndTime = this.lastMarketSlotMap
+			.get(l2Args.marketType)
+			.get(l2Args.marketIndex);
+		if (!lastMarketSlotAndTime) {
+			this.lastMarketSlotMap
+				.get(l2Args.marketType)
+				.set(l2Args.marketIndex, { slot, ts: Date.now() });
+		}
+
 		if (slot) {
 			delete l2.slot;
 		}
@@ -127,6 +145,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
 			)
 				return;
 		}
+
 		this.lastSeenL2Formatted
 			.get(l2Args.marketType)
 			?.set(l2Args.marketIndex, JSON.stringify(l2Formatted));
@@ -148,6 +167,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
 			l2Args.marketIndex
 		);
 
+		// Check if slot diffs are too large for oracle
 		if (
 			Math.abs(slot - parseInt(l2Formatted['oracleData']['slot'])) >
 			this.killSwitchSlotDiffThreshold
@@ -155,9 +175,34 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
 			console.log(`Killing process due to slot diffs for market ${marketName}: 
 				dlobProvider slot: ${slot}
 				oracle slot: ${l2Formatted['oracleData']['slot']}
+			`);
+			process.exit(1);
+		}
+
+		// Check if times and slots are too different for market
+		if (
+			lastMarketSlotAndTime &&
+			l2Formatted['marketSlot'] === lastMarketSlotAndTime.slot &&
+			Date.now() - lastMarketSlotAndTime.ts > MAKRET_STALENESS_THRESHOLD
+		) {
+			console.log(`Killing process due to same slot for market ${marketName} after > ${MAKRET_STALENESS_THRESHOLD}ms: 
+				dlobProvider slot: ${slot}
 				market slot: ${l2Formatted['marketSlot']}
 			`);
 			process.exit(1);
+		} else if (
+			lastMarketSlotAndTime &&
+			l2Formatted['marketSlot'] !== lastMarketSlotAndTime.slot
+		) {
+			console.log(
+				`Updating market slot for ${l2Args.marketName} with slot ${l2Formatted['marketSlot']}`
+			);
+			this.lastMarketSlotMap
+				.get(l2Args.marketType)
+				.set(l2Args.marketIndex, {
+					slot: l2Formatted['marketSlot'],
+					ts: Date.now(),
+				});
 		}
 
 		const l2Formatted_depth100 = Object.assign({}, l2Formatted, {

yarn.lock

@@ -115,10 +115,10 @@
     enabled "2.0.x"
     kuler "^2.0.0"
 
-"@drift-labs/sdk@2.59.0-beta.0":
+"@drift-labs/sdk@2.59.0-beta.5":
-  version "2.59.0-beta.0"
+  version "2.59.0-beta.5"
-  resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.59.0-beta.0.tgz#e71a862ba67851522e24e9c89512aef6406eef29"
+  resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.59.0-beta.5.tgz#50c8bd135c0bcdff224a0277841555433ced218a"
-  integrity sha512-KmOXLbp+x2Q3HO7LppCkFO/suTevDxFVVcyfGLNonuHQLGoUUIKQTF+j87fg5IVem8a8dhqo/DyzM6mQxO6iGg==
+  integrity sha512-RgVwGUbuQaZgw4LLe2yxBXiWDnzleYU7lr+Vjq0EFi7hNeyJ35V+IiOfDxA9G9/D3vv1BfXD3adx0gR2rLTUBg==
   dependencies:
     "@coral-xyz/anchor" "0.28.1-beta.2"
     "@ellipsis-labs/phoenix-sdk" "^1.4.2"