Merge pull request #82 from drift-labs/master

market kill switch

package.json

@@ -6,7 +6,7 @@
   "license": "Apache-2.0",
   "dependencies": {
     "@coral-xyz/anchor": "^0.29.0",
-    "@drift-labs/sdk": "2.58.0-beta.0",
+    "@drift-labs/sdk": "2.59.0-beta.5",
     "@opentelemetry/api": "^1.1.0",
     "@opentelemetry/auto-instrumentations-node": "^0.31.1",
     "@opentelemetry/exporter-prometheus": "^0.31.0",

src/dlob-subscriber/DLOBSubscriberIO.ts

@@ -27,6 +27,8 @@ type wsMarketL2Args = {
 	updateOnChange?: boolean;
 };
 
+const MAKRET_STALENESS_THRESHOLD = 10 * 60 * 1000;
+
 export type wsMarketInfo = {
 	marketIndex: number;
 	marketName: string;
@@ -37,6 +39,10 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
 	public lastSeenL2Formatted: Map<MarketType, Map<number, any>>;
 	redisClient: RedisClient;
 	public killSwitchSlotDiffThreshold: number;
+	public lastMarketSlotMap: Map<
+		MarketType,
+		Map<number, { slot: number; ts: number }>
+	>;
 
 	constructor(
 		config: DLOBSubscriptionConfig & {
@@ -56,16 +62,24 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
 		this.lastSeenL2Formatted = new Map();
 		this.lastSeenL2Formatted.set(MarketType.SPOT, new Map());
 		this.lastSeenL2Formatted.set(MarketType.PERP, new Map());
+		this.lastMarketSlotMap = new Map();
+		this.lastMarketSlotMap.set(MarketType.SPOT, new Map());
+		this.lastMarketSlotMap.set(MarketType.PERP, new Map());
 
 		for (const market of config.perpMarketInfos) {
+			const perpMarket = this.driftClient.getPerpMarketAccount(
+				market.marketIndex
+			);
+			const includeVamm = !isVariant(perpMarket.status, 'ammPaused');
+
 			this.marketL2Args.push({
 				marketIndex: market.marketIndex,
 				marketType: MarketType.PERP,
 				marketName: market.marketName,
 				depth: -1,
 				numVammOrders: 100,
-				includeVamm: true,
-				updateOnChange: true,
+				includeVamm,
+				updateOnChange: includeVamm,
 				fallbackL2Generators: [],
 			});
 		}
@@ -102,6 +116,15 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
 		const { marketName, ...l2FuncArgs } = l2Args;
 		const l2 = this.getL2(l2FuncArgs);
 		const slot = l2.slot;
+		const lastMarketSlotAndTime = this.lastMarketSlotMap
+			.get(l2Args.marketType)
+			.get(l2Args.marketIndex);
+		if (!lastMarketSlotAndTime) {
+			this.lastMarketSlotMap
+				.get(l2Args.marketType)
+				.set(l2Args.marketIndex, { slot, ts: Date.now() });
+		}
+
 		if (slot) {
 			delete l2.slot;
 		}
@@ -122,6 +145,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
 			)
 				return;
 		}
+
 		this.lastSeenL2Formatted
 			.get(l2Args.marketType)
 			?.set(l2Args.marketIndex, JSON.stringify(l2Formatted));
@@ -143,6 +167,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
 			l2Args.marketIndex
 		);
 
+		// Check if slot diffs are too large for oracle
 		if (
 			Math.abs(slot - parseInt(l2Formatted['oracleData']['slot'])) >
 			this.killSwitchSlotDiffThreshold
@@ -150,9 +175,34 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
 			console.log(`Killing process due to slot diffs for market ${marketName}: 
 				dlobProvider slot: ${slot}
 				oracle slot: ${l2Formatted['oracleData']['slot']}
+			`);
+			process.exit(1);
+		}
+
+		// Check if times and slots are too different for market
+		if (
+			lastMarketSlotAndTime &&
+			l2Formatted['marketSlot'] === lastMarketSlotAndTime.slot &&
+			Date.now() - lastMarketSlotAndTime.ts > MAKRET_STALENESS_THRESHOLD
+		) {
+			console.log(`Killing process due to same slot for market ${marketName} after > ${MAKRET_STALENESS_THRESHOLD}ms: 
+				dlobProvider slot: ${slot}
 				market slot: ${l2Formatted['marketSlot']}
 			`);
 			process.exit(1);
+		} else if (
+			lastMarketSlotAndTime &&
+			l2Formatted['marketSlot'] !== lastMarketSlotAndTime.slot
+		) {
+			console.log(
+				`Updating market slot for ${l2Args.marketName} with slot ${l2Formatted['marketSlot']}`
+			);
+			this.lastMarketSlotMap
+				.get(l2Args.marketType)
+				.set(l2Args.marketIndex, {
+					slot: l2Formatted['marketSlot'],
+					ts: Date.now(),
+				});
 		}
 
 		const l2Formatted_depth100 = Object.assign({}, l2Formatted, {

src/index.ts

@@ -116,21 +116,23 @@ app.use(compression());
 app.set('trust proxy', 1);
 app.use(logHttp);
 app.use(handleResponseTime);
-app.use(
-	rateLimit({
-		windowMs: 1000, // 1 second
-		max: rateLimitCallsPerSecond,
-		standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
-		legacyHeaders: false, // Disable the `X-RateLimit-*` headers
-		skip: (req, _res) => {
-			if (!loadTestAllowed) {
-				return false;
-			}
+if (!FEATURE_FLAGS.DISABLE_RATE_LIMIT) {
+	app.use(
+		rateLimit({
+			windowMs: 1000, // 1 second
+			max: rateLimitCallsPerSecond,
+			standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers
+			legacyHeaders: false, // Disable the `X-RateLimit-*` headers
+			skip: (req, _res) => {
+				if (!loadTestAllowed) {
+					return false;
+				}
 
-			return req.headers['user-agent'].includes('k6');
-		},
-	})
-);
+				return req.headers['user-agent'].includes('k6');
+			},
+		})
+	);
+}
 
 // strip off /dlob, if the request comes from exchange history server LB
 app.use((req, _res, next) => {

src/utils/featureFlags.ts

@@ -12,6 +12,10 @@ export const FEATURE_FLAGS = {
 	DISABLE_GPA_REFRESH: process.env.DISABLE_GPA_REFRESH
 		? process.env.DISABLE_GPA_REFRESH.toLowerCase() === 'true'
 		: false,
+
+	DISABLE_RATE_LIMIT: process.env.DISABLE_RATE_LIMIT
+		? process.env.DISABLE_RATE_LIMIT.toLowerCase() === 'true'
+		: false,
 };
 
 export default FEATURE_FLAGS;

yarn.lock

@@ -115,10 +115,10 @@
     enabled "2.0.x"
     kuler "^2.0.0"
 
-"@drift-labs/sdk@2.58.0-beta.0":
-  version "2.58.0-beta.0"
-  resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.58.0-beta.0.tgz#853ec219045018b102d5c6d6c33c4caf197d1924"
-  integrity sha512-+NS4/cKLrrNtMVRhfpJJK1d1KlbX5xYmvZPYDKXmKCQ5GAaPyBgwF+0S+ZMmGb2j1MsHPXhjJKsbI8QTQhUAGg==
+"@drift-labs/sdk@2.59.0-beta.5":
+  version "2.59.0-beta.5"
+  resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.59.0-beta.5.tgz#50c8bd135c0bcdff224a0277841555433ced218a"
+  integrity sha512-RgVwGUbuQaZgw4LLe2yxBXiWDnzleYU7lr+Vjq0EFi7hNeyJ35V+IiOfDxA9G9/D3vv1BfXD3adx0gR2rLTUBg==
   dependencies:
     "@coral-xyz/anchor" "0.28.1-beta.2"
     "@ellipsis-labs/phoenix-sdk" "^1.4.2"