feat: TOB monitoring logic

.husky/pre-commit

@@ -0,0 +1,22 @@
+#!/usr/bin/env sh
+. "$(dirname -- "$0")/_/husky.sh"
+
+# Pre-commit hook to check for potential secrets
+echo "🔍 Running secret check..."
+
+# Run the secret checking script
+if [ -f "scripts/check-secrets.sh" ]; then
+    ./scripts/check-secrets.sh
+    if [ $? -ne 0 ]; then
+        echo "❌ Pre-commit hook failed. Please fix the issues above before committing."
+        echo "💡 If these are false positives, you can:"
+        echo "   1. Add the file to .gitignore"
+        echo "   2. Use placeholder values instead of real secrets"
+        echo "   3. Bypass the hook with: git commit --no-verify"
+        exit 1
+    fi
+else
+    echo "⚠️  Warning: check-secrets.sh not found"
+fi
+
+echo "✅ Pre-commit checks passed" 

README.md

@@ -14,51 +14,82 @@
 
 This is the backend server that provides a REST API for the drift [DLOB](https://docs.drift.trade/about-v2/decentralized-orderbook).
 
+## Features
+
+- Real-time DLOB data publishing
+- Support for both perp and spot markets
+- Multiple subscription methods (WebSocket, gRPC, polling)
+- Health checks and metrics
+- **TOB (Top of Book) monitoring for stuck orders** (see [TOB Monitoring Configuration](#tob-monitoring-configuration))
+
 # Run the server
 
 ## Setup
 
 The build dependencies
+
 ```
 git submodule update --init
 bash build_all.sh
 ```
 
 First set the necessary environment variables:
+
 ```
 cp .env.example .env
 ```
 
-## Environment Configuration
+## Security
+
+### Pre-commit Hook
+
+This repository uses [Husky](https://typicode.github.io/husky/) to manage Git hooks. A pre-commit hook automatically checks for potential secrets before each commit. The hook will:
+
+- Scan for RPC URLs, API keys, tokens, and other potential secrets
+- Prevent commits that contain suspicious patterns
+- Provide helpful guidance if false positives are detected
+
+The hook is automatically installed when you run `npm install` (via the `prepare` script). If you need to bypass it for a specific commit, use:
+
+```bash
+git commit --no-verify
+```
+
+### Scripts
+
+The `scripts/` directory contains utility scripts:
+
+- `check-secrets.sh` - Secret detection script used by the pre-commit hook
+
+## Environment Variables
 
 To properly configure the DLOB server, set the following environment variables in your `.env` file:
 
-| Variable                  | Description                                               | Example Value                                                      |
+| Variable                      | Description                                                     | Example Value                       |
-|---------------------------|-----------------------------------------------------------|--------------------------------------------------------------------|
+| ----------------------------- | --------------------------------------------------------------- | ----------------------------------- |
-| `ENDPOINT`                | The Solana RPC node http endpoint.                        | `https://your-private-rpc-node.com`                                |
+| `ENDPOINT`                    | The Solana RPC node http endpoint.                              | `https://your-private-rpc-node.com` |
-| `WS_ENDPOINT`             | The Solana RPC node websocket endpoint.                   | `wss://your-private-rpc-node.com`                                  |
+| `WS_ENDPOINT`                 | The Solana RPC node websocket endpoint.                         | `wss://your-private-rpc-node.com`   |
-| `USE_WEBSOCKET`           | Flag to enable WebSocket connection.                      | `true`                                                             |
+| `USE_WEBSOCKET`               | Flag to enable WebSocket connection.                            | `true`                              |
-| `USE_ORDER_SUBSCRIBER`    | Flag to enable order subscriber DLOB source.              | `true`                                                             |
+| `USE_ORDER_SUBSCRIBER`        | Flag to enable order subscriber DLOB source.                    | `true`                              |
-| `DISABLE_GPA_REFRESH`     | Flag to disable periodic refresh using `getProgramAccounts`. | `true`                                                          |
+| `DISABLE_GPA_REFRESH`         | Flag to disable periodic refresh using `getProgramAccounts`.    | `true`                              |
-| `ENV`                     | The network environment the server is connecting to.      | `mainnet-beta`                                                     |
+| `ENV`                         | The network environment the server is connecting to.            | `mainnet-beta`                      |
-| `PORT`                    | The port number the HTTP server listens on.               | `6969`                                                             |
+| `PORT`                        | The port number the HTTP server listens on.                     | `6969`                              |
-| `METRICS_PORT`            | The port number for Prometheus metrics.                   | `9465`                                                             |
+| `METRICS_PORT`                | The port number for Prometheus metrics.                         | `9465`                              |
-| `PRIVATE_KEY`             | Path to the Solana private key file.                      | `/path/to/keypair.json`                                            |
+| `PRIVATE_KEY`                 | Path to the Solana private key file.                            | `/path/to/keypair.json`             |
-| `RATE_LIMIT_CALLS_PER_SECOND` | Maximum number of API calls per second.               | `100`                                                              |
+| `RATE_LIMIT_CALLS_PER_SECOND` | Maximum number of API calls per second.                         | `100`                               |
-| `PERP_MARKETS_TO_LOAD`    | Number of perpetual markets to load at startup.           | `0`                                                                |
+| `PERP_MARKETS_TO_LOAD`        | Number of perpetual markets to load at startup.                 | `0`                                 |
-| `SPOT_MARKETS_TO_LOAD`    | Number of spot markets to load at startup.                | `5`                                                                |
+| `SPOT_MARKETS_TO_LOAD`        | Number of spot markets to load at startup.                      | `5`                                 |
-| `ELASTICACHE_HOST`        | (for websocket server) Redis host endpoint.               | `localhost`                                                        |
+| `ELASTICACHE_HOST`            | (for websocket server) Redis host endpoint.                     | `localhost`                         |
-| `ELASTICACHE_PORT`        | (for websocket server) Redis port.                        | `6379`                                                             |
+| `ELASTICACHE_PORT`            | (for websocket server) Redis port.                              | `6379`                              |
-| `REDIS_CLIENT`            | (for websocket server) Redis client type (DLOB/DLOB_HELIUS).| `DLOB`                                                           |
+| `REDIS_CLIENT`                | (for websocket server) Redis client type (DLOB/DLOB_HELIUS).    | `DLOB`                              |
-| `WS_PORT`                 | (for websocket server) The port to run the websocket server on.|  `3000`                                                       |
+| `WS_PORT`                     | (for websocket server) The port to run the websocket server on. | `3000`                              |
-
 
 Note: multiple Redis hosts can be provided by providing a comma separated string.
 
-
 ## HTTP mode
 
 The HTTP server as documented [here](https://drift-labs.github.io/v2-teacher/?python#orderbook-trades-dlob-server) can be run with, and by default accessible on `http://127.0.0.1:6969`:
+
 ```
 yarn run dev
 ```
@@ -68,22 +99,26 @@ yarn run dev
 The websocket server has 2 components, the `dlob-publisher` that takes frequent snapshots of the DLOB and publishes them to Redis, and `ws-manager` listens for new connections and sends the latest DLOB to ws clients, the two components communicate through Redis pub-sub.
 
 To run the websocket server, a Redis cache is required, and the following environment variables must be set:
-* `REDIS_HOSTS`
+
-* `REDIS_PASSWORDS`
+- `REDIS_HOSTS`
-* `REDIS_PORTS`
+- `REDIS_PASSWORDS`
+- `REDIS_PORTS`
 
 In the first terminal, start the redis cluster:
+
 ```
 bash redisCluster.sh start
 bash redisCluster.sh create
 ```
 
 In second terminal, run:
+
 ```
 yarn run dlob-publish
 ```
 
 In a third terminal, run:
+
 ```
 yarn run ws-manager
 ```
@@ -91,6 +126,7 @@ yarn run ws-manager
 Then connect to the ws server at ws://127.0.0.1:3000
 
 When you're done, stop the redis cluster:
+
 ```
 bash redisCluster.sh stop
 ```
@@ -100,3 +136,32 @@ bash redisCluster.sh stop
 Documentation for connecting to the dlob server are available [here](https://drift-labs.github.io/v2-teacher/?python#orderbook-trades-dlob-server)
 
 TODO: complete client examples.
+
+## TOB (Top of Book) Monitoring [#tob-monitoring]
+
+The server includes a TOB monitoring feature that detects when order books become stuck due to ghost/stuck orders. This is particularly useful for gRPC connections that may miss updates.
+
+### Configuration
+
+Set the following environment variables to enable and configure TOB monitoring:
+
+- `ENABLE_TOB_MONITORING=true` - Enable TOB monitoring (default: false)
+- `TOB_CHECK_INTERVAL=60000` - How often to check TOB (default: 60 seconds)
+- `TOB_STUCK_THRESHOLD=60000` - How long TOB can be stuck before resubscribing (default: 60 seconds)
+- `TOP_MONITORING_ENABLED_PERP_MARKETS=0,1,2` - Comma-separated list of perp market indexes to monitor for TOB (default: 0,1,2 for SOL-PERP, BTC-PERP, ETH-PERP)
+
+### How it works
+
+1. Checks if the current node is configured to handle any TOB monitoring markets
+2. Only enables monitoring if the node has TOB monitoring markets configured
+3. Monitors the top bid/ask prices for TOB monitoring perp markets on this node
+4. If TOB hasn't changed for the configured threshold time, triggers a resubscribe
+5. Performs unsubscribe → subscribe → fetch sequence on the OrderSubscriber instance
+6. Logs warnings and updates metrics for monitoring
+
+### Metrics
+
+The following metrics are available for TOB monitoring:
+
+- `tob_resubscribe` - Counter for resubscribe attempts (with success/failure labels)
+- `tob_stuck_duration` - Gauge showing how long TOB has been stuck for each market

scripts/check-secrets.sh

@@ -0,0 +1,145 @@
+#!/bin/bash
+
+# Pre-commit hook to check for potential secrets
+# This script checks for common patterns that might indicate secrets being committed
+
+set -e
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+echo "🔍 Checking for potential secrets in staged files..."
+
+# Get list of staged files
+STAGED_FILES=$(git diff --cached --name-only --diff-filter=ACM)
+
+if [ -z "$STAGED_FILES" ]; then
+    echo "✅ No files staged for commit"
+    exit 0
+fi
+
+# Patterns to check for (add more as needed)
+PATTERNS=(
+    # RPC URLs and endpoints
+    "https://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"
+    "wss://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"
+    "ws://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"
+    "http://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}"
+    
+    # API keys and tokens (common patterns)
+    "[a-zA-Z0-9]{32,}"  # Long alphanumeric strings (likely tokens)
+    "sk-[a-zA-Z0-9]{20,}"  # Stripe-like keys
+    "pk_[a-zA-Z0-9]{20,}"  # Public keys
+    "sk_[a-zA-Z0-9]{20,}"  # Secret keys
+    
+    # Private keys and addresses
+    "[1-9A-HJ-NP-Za-km-z]{32,}"  # Base58 encoded strings (likely private keys)
+    "0x[a-fA-F0-9]{40}"  # Ethereum addresses
+    "[1-9A-HJ-NP-Za-km-z]{44}"  # Solana addresses
+    
+    # Common secret patterns
+    "password.*=.*['\"][^'\"]{8,}['\"]"
+    "secret.*=.*['\"][^'\"]{8,}['\"]"
+    "token.*=.*['\"][^'\"]{8,}['\"]"
+    "key.*=.*['\"][^'\"]{8,}['\"]"
+    "api_key.*=.*['\"][^'\"]{8,}['\"]"
+    "private_key.*=.*['\"][^'\"]{8,}['\"]"
+    
+    # Environment variables that might contain secrets
+    "ENDPOINT.*=.*['\"][^'\"]{10,}['\"]"
+    "WS_ENDPOINT.*=.*['\"][^'\"]{10,}['\"]"
+    "GRPC_ENDPOINT.*=.*['\"][^'\"]{10,}['\"]"
+    "TOKEN.*=.*['\"][^'\"]{8,}['\"]"
+    "API_KEY.*=.*['\"][^'\"]{8,}['\"]"
+    "SECRET.*=.*['\"][^'\"]{8,}['\"]"
+)
+
+# Files to exclude from checking
+EXCLUDE_PATTERNS=(
+    "node_modules"
+    ".git"
+    "*.lock"
+    "*.lockb"
+    "*.so"
+    "*.dll"
+    "*.exe"
+    "*.bin"
+    "*.min.js"
+    "*.min.css"
+    "*.map"
+    "*.log"
+    "*.tmp"
+    "*.cache"
+    "build/"
+    "dist/"
+    ".env.example"
+    "scripts/check-secrets.sh"  # Exclude this script itself
+)
+
+# Function to check if file should be excluded
+should_exclude() {
+    local file="$1"
+    for pattern in "${EXCLUDE_PATTERNS[@]}"; do
+        if [[ "$file" == *"$pattern"* ]] || [[ "$file" =~ $pattern ]]; then
+            return 0
+        fi
+    done
+    return 1
+}
+
+# Function to check for patterns in a file
+check_file() {
+    local file="$1"
+    local found_secrets=false
+    
+    # Skip binary files
+    if file "$file" | grep -q "binary"; then
+        return 0
+    fi
+    
+    # Check each pattern
+    for pattern in "${PATTERNS[@]}"; do
+        if git grep --cached "$pattern" "$file" >/dev/null 2>&1; then
+            echo -e "${RED}⚠️  Potential secret found in $file${NC}"
+            echo -e "${YELLOW}Pattern: $pattern${NC}"
+            git grep --cached "$pattern" "$file" | head -3
+            echo ""
+            found_secrets=true
+        fi
+    done
+    
+    if [ "$found_secrets" = true ]; then
+        return 1
+    fi
+    
+    return 0
+}
+
+# Check each staged file
+has_secrets=false
+for file in $STAGED_FILES; do
+    if should_exclude "$file"; then
+        echo -e "${GREEN}✅ Skipping excluded file: $file${NC}"
+        continue
+    fi
+    
+    if ! check_file "$file"; then
+        has_secrets=true
+    fi
+done
+
+if [ "$has_secrets" = true ]; then
+    echo -e "${RED}❌ Potential secrets detected!${NC}"
+    echo -e "${YELLOW}Please review the above matches and ensure no real secrets are being committed.${NC}"
+    echo -e "${YELLOW}If these are false positives, you can:${NC}"
+    echo -e "${YELLOW}1. Add the file to .gitignore${NC}"
+    echo -e "${YELLOW}2. Use placeholder values instead of real secrets${NC}"
+    echo -e "${YELLOW}3. Add the pattern to the exclude list in this script${NC}"
+    exit 1
+else
+    echo -e "${GREEN}✅ No potential secrets detected${NC}"
+    exit 0
+fi 

src/publishers/dlobPublisher.ts

@@ -87,6 +87,14 @@ const kinesisRecordsSentCounter = metricsV2.addCounter(
 	'kinesis_records_sent',
 	'Number of records sent to Kinesis'
 );
+const tobResubscribeCounter = metricsV2.addCounter(
+	'tob_resubscribe',
+	'Number of TOB resubscribes triggered'
+);
+const tobStuckGauge = metricsV2.addGauge(
+	'tob_stuck_duration',
+	'Duration TOB has been stuck for each market'
+);
 metricsV2.finalizeObservables();
 
 //@ts-ignore
@@ -94,7 +102,7 @@ const sdkConfig = initialize({ env: process.env.ENV });
 let driftClient: DriftClient;
 
 const opts = program.opts();
-setLogLevel(opts.debug ? 'debug' : 'info');
+setLogLevel('debug');
 
 const useGrpc = process.env.USE_GRPC?.toLowerCase() === 'true';
 const useWebsocket = process.env.USE_WEBSOCKET?.toLowerCase() === 'true';
@@ -116,6 +124,16 @@ const WS_FALLBACK_FETCH_INTERVAL = 60_000;
 const KILLSWITCH_SLOT_DIFF_THRESHOLD =
 	parseInt(process.env.KILLSWITCH_SLOT_DIFF_THRESHOLD) || 200;
 
+// TOB monitoring configuration
+const ENABLE_TOB_MONITORING =
+	process.env.ENABLE_TOB_MONITORING?.toLowerCase() === 'true';
+const TOB_CHECK_INTERVAL = parseInt(process.env.TOB_CHECK_INTERVAL) || 60_000; // 1 minute
+const TOB_STUCK_THRESHOLD = parseInt(process.env.TOB_STUCK_THRESHOLD) || 60_000; // 1 minute without change
+const TOB_MONITORING_ENABLED_PERP_MARKETS = process.env
+	.TOB_MONITORING_ENABLED_PERP_MARKETS
+	? parsePositiveIntArray(process.env.TOB_MONITORING_ENABLED_PERP_MARKETS)
+	: [0, 1, 2]; // Default to SOL-PERP, BTC-PERP, ETH-PERP
+
 // comma separated list of perp market indexes to load: i.e. 0,1,2,3
 const PERP_MARKETS_TO_LOAD =
 	process.env.PERP_MARKETS_TO_LOAD !== undefined
@@ -142,6 +160,16 @@ logger.info(
 );
 logger.info(`DriftEnv:     ${driftEnv}`);
 logger.info(`Commit:       ${commitHash}`);
+logger.info(
+	`TOB Monitoring: ${ENABLE_TOB_MONITORING ? 'enabled' : 'disabled'}`
+);
+if (ENABLE_TOB_MONITORING) {
+	logger.info(`TOB Check Interval: ${TOB_CHECK_INTERVAL}ms`);
+	logger.info(`TOB Stuck Threshold: ${TOB_STUCK_THRESHOLD}ms`);
+	logger.info(
+		`TOB Monitoring Markets: ${TOB_MONITORING_ENABLED_PERP_MARKETS.join(', ')}`
+	);
+}
 
 let MARKET_SUBSCRIBERS: SubscriberLookup = {};
 
@@ -437,6 +465,8 @@ const main = async () => {
 	);
 
 	let dlobProvider: DLOBProvider;
+	let orderSubscriber: OrderSubscriberFiltered | undefined;
+
 	if (useOrderSubscriber) {
 		let subscriptionConfig: any = {
 			type: 'polling',
@@ -480,7 +510,7 @@ const main = async () => {
 			};
 		}
 
-		const orderSubscriber = new OrderSubscriberFiltered({
+		orderSubscriber = new OrderSubscriberFiltered({
 			driftClient,
 			subscriptionConfig,
 			ignoreList,
@@ -623,6 +653,158 @@ const main = async () => {
 		});
 	}, 10_000);
 
+	// TOB monitoring for configured perp markets to detect stuck orders
+	// Check if this node has any TOB monitoring markets configured
+	const tobMonitoringMarketsInThisNode =
+		TOB_MONITORING_ENABLED_PERP_MARKETS.filter((marketIndex) =>
+			perpMarketInfos.some((market) => market.marketIndex === marketIndex)
+		);
+
+	const shouldEnableTobMonitoring =
+		ENABLE_TOB_MONITORING &&
+		useOrderSubscriber &&
+		tobMonitoringMarketsInThisNode.length > 0;
+
+	// Track last TOB update times and values for each TOB monitoring market
+	const lastTobUpdateTimes = new Map<number, number>();
+	const lastTobValues = new Map<number, { bid: string; ask: string }>();
+
+	// Initialize TOB tracking for TOB monitoring markets in this node
+	tobMonitoringMarketsInThisNode.forEach((marketIndex) => {
+		lastTobUpdateTimes.set(marketIndex, Date.now());
+		lastTobValues.set(marketIndex, { bid: '', ask: '' });
+	});
+
+	// Log TOB monitoring status
+	if (ENABLE_TOB_MONITORING) {
+		logger.info(
+			`TOB Monitoring Markets in this node: ${tobMonitoringMarketsInThisNode.join(
+				', '
+			)}`
+		);
+		logger.info(
+			`TOB Monitoring active: ${shouldEnableTobMonitoring ? 'yes' : 'no'}`
+		);
+	}
+
+	// TOB monitoring function
+	const checkTobForStuckOrders = async () => {
+		if (!shouldEnableTobMonitoring) {
+			return; // Only monitor when using OrderSubscriber, TOB monitoring is enabled, and node has major markets
+		}
+
+		logger.debug('Starting TOB monitoring check');
+
+		const currentTime = Date.now();
+
+		for (const marketIndex of tobMonitoringMarketsInThisNode) {
+			try {
+				// Get current TOB from DLOB
+				const slot = slotSource.getSlot();
+				const dlob = await dlobProvider.getDLOB(slot);
+
+				// Get oracle data for the market
+				const oracleData = driftClient.getOracleDataForPerpMarket(marketIndex);
+
+				// Get L2 orderbook to check TOB
+				const l2OrderBook = dlob.getL2({
+					marketIndex,
+					marketType: { perp: {} },
+					slot,
+					oraclePriceData: oracleData,
+					depth: 1, // Only need top level
+				});
+
+				const bestBid = l2OrderBook.bids[0]?.price;
+				const bestAsk = l2OrderBook.asks[0]?.price;
+
+				if (bestBid && bestAsk) {
+					const currentTobValues = {
+						bid: bestBid.toString(),
+						ask: bestAsk.toString(),
+					};
+					const lastTobValue = lastTobValues.get(marketIndex);
+					const lastUpdate = lastTobUpdateTimes.get(marketIndex);
+
+					// Check if TOB has changed
+					const tobChanged =
+						!lastTobValue ||
+						lastTobValue.bid !== currentTobValues.bid ||
+						lastTobValue.ask !== currentTobValues.ask;
+
+					if (tobChanged) {
+						// TOB changed, update tracking
+						lastTobValues.set(marketIndex, currentTobValues);
+						lastTobUpdateTimes.set(marketIndex, currentTime);
+						logger.debug(
+							`TOB updated for market ${marketIndex}: bid=${currentTobValues.bid}, ask=${currentTobValues.ask}`
+						);
+					} else if (
+						lastUpdate &&
+						currentTime - lastUpdate > TOB_STUCK_THRESHOLD
+					) {
+						// TOB has been stuck for too long, trigger resubscribe
+						const stuckDuration = (currentTime - lastUpdate) / 1000;
+						logger.warn(
+							`TOB stuck for market ${marketIndex} for ${stuckDuration}s, triggering resubscribe`
+						);
+
+						// Update metrics
+						tobStuckGauge.setLatestValue(stuckDuration, {
+							marketIndex: marketIndex.toString(),
+							marketType: 'perp',
+						});
+
+						// Get the OrderSubscriber instance for resubscribe
+						if (orderSubscriber) {
+							try {
+								// Resubscribe and fetch to clear stuck state
+								await orderSubscriber.unsubscribe();
+								await orderSubscriber.subscribe();
+								await orderSubscriber.fetch();
+
+								logger.info(
+									`Successfully resubscribed OrderSubscriber for market ${marketIndex}`
+								);
+
+								// Update metrics
+								tobResubscribeCounter.add(1, {
+									marketIndex: marketIndex.toString(),
+									marketType: 'perp',
+									success: 'true',
+								});
+
+								// Reset the timer after successful resubscribe
+								lastTobUpdateTimes.set(marketIndex, currentTime);
+							} catch (error) {
+								logger.error(
+									`Failed to resubscribe OrderSubscriber for market ${marketIndex}:`,
+									error
+								);
+
+								// Update metrics for failed resubscribe
+								tobResubscribeCounter.add(1, {
+									marketIndex: marketIndex.toString(),
+									marketType: 'perp',
+									success: 'false',
+								});
+							}
+						} else {
+							logger.error(
+								`OrderSubscriber not available for market ${marketIndex}`
+							);
+						}
+					}
+				}
+			} catch (error) {
+				logger.error(`Error checking TOB for market ${marketIndex}:`, error);
+			}
+		}
+	};
+
+	// Start TOB monitoring
+	setInterval(checkTobForStuckOrders, TOB_CHECK_INTERVAL);
+
 	const handleStartup = async (_req, res, _next) => {
 		if (driftClient.isSubscribed && dlobProvider.size() > 0) {
 			res.writeHead(200);