diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100755 index 0000000..fe7ea7b --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1,22 @@ +#!/usr/bin/env sh +. "$(dirname -- "$0")/_/husky.sh" + +# Pre-commit hook to check for potential secrets +echo "🔍 Running secret check..." + +# Run the secret checking script +if [ -f "scripts/check-secrets.sh" ]; then + ./scripts/check-secrets.sh + if [ $? -ne 0 ]; then + echo "❌ Pre-commit hook failed. Please fix the issues above before committing." + echo "💡 If these are false positives, you can:" + echo " 1. Add the file to .gitignore" + echo " 2. Use placeholder values instead of real secrets" + echo " 3. Bypass the hook with: git commit --no-verify" + exit 1 + fi +else + echo "⚠️ Warning: check-secrets.sh not found" +fi + +echo "✅ Pre-commit checks passed" \ No newline at end of file diff --git a/README.md b/README.md index aeb0aac..4066f58 100644 --- a/README.md +++ b/README.md @@ -14,51 +14,82 @@ This is the backend server that provides a REST API for the drift [DLOB](https://docs.drift.trade/about-v2/decentralized-orderbook). +## Features + +- Real-time DLOB data publishing +- Support for both perp and spot markets +- Multiple subscription methods (WebSocket, gRPC, polling) +- Health checks and metrics +- **TOB (Top of Book) monitoring for stuck orders** (see [TOB Monitoring Configuration](#tob-monitoring-configuration)) + # Run the server ## Setup The build dependencies + ``` git submodule update --init bash build_all.sh ``` First set the necessary environment variables: + ``` cp .env.example .env ``` -## Environment Configuration +## Security + +### Pre-commit Hook + +This repository uses [Husky](https://typicode.github.io/husky/) to manage Git hooks. A pre-commit hook automatically checks for potential secrets before each commit. The hook will: + +- Scan for RPC URLs, API keys, tokens, and other potential secrets +- Prevent commits that contain suspicious patterns +- Provide helpful guidance if false positives are detected + +The hook is automatically installed when you run `npm install` (via the `prepare` script). If you need to bypass it for a specific commit, use: + +```bash +git commit --no-verify +``` + +### Scripts + +The `scripts/` directory contains utility scripts: + +- `check-secrets.sh` - Secret detection script used by the pre-commit hook + +## Environment Variables To properly configure the DLOB server, set the following environment variables in your `.env` file: -| Variable | Description | Example Value | -|---------------------------|-----------------------------------------------------------|--------------------------------------------------------------------| -| `ENDPOINT` | The Solana RPC node http endpoint. | `https://your-private-rpc-node.com` | -| `WS_ENDPOINT` | The Solana RPC node websocket endpoint. | `wss://your-private-rpc-node.com` | -| `USE_WEBSOCKET` | Flag to enable WebSocket connection. | `true` | -| `USE_ORDER_SUBSCRIBER` | Flag to enable order subscriber DLOB source. | `true` | -| `DISABLE_GPA_REFRESH` | Flag to disable periodic refresh using `getProgramAccounts`. | `true` | -| `ENV` | The network environment the server is connecting to. | `mainnet-beta` | -| `PORT` | The port number the HTTP server listens on. | `6969` | -| `METRICS_PORT` | The port number for Prometheus metrics. | `9465` | -| `PRIVATE_KEY` | Path to the Solana private key file. | `/path/to/keypair.json` | -| `RATE_LIMIT_CALLS_PER_SECOND` | Maximum number of API calls per second. | `100` | -| `PERP_MARKETS_TO_LOAD` | Number of perpetual markets to load at startup. | `0` | -| `SPOT_MARKETS_TO_LOAD` | Number of spot markets to load at startup. | `5` | -| `ELASTICACHE_HOST` | (for websocket server) Redis host endpoint. | `localhost` | -| `ELASTICACHE_PORT` | (for websocket server) Redis port. | `6379` | -| `REDIS_CLIENT` | (for websocket server) Redis client type (DLOB/DLOB_HELIUS).| `DLOB` | -| `WS_PORT` | (for websocket server) The port to run the websocket server on.| `3000` | - +| Variable | Description | Example Value | +| ----------------------------- | --------------------------------------------------------------- | ----------------------------------- | +| `ENDPOINT` | The Solana RPC node http endpoint. | `https://your-private-rpc-node.com` | +| `WS_ENDPOINT` | The Solana RPC node websocket endpoint. | `wss://your-private-rpc-node.com` | +| `USE_WEBSOCKET` | Flag to enable WebSocket connection. | `true` | +| `USE_ORDER_SUBSCRIBER` | Flag to enable order subscriber DLOB source. | `true` | +| `DISABLE_GPA_REFRESH` | Flag to disable periodic refresh using `getProgramAccounts`. | `true` | +| `ENV` | The network environment the server is connecting to. | `mainnet-beta` | +| `PORT` | The port number the HTTP server listens on. | `6969` | +| `METRICS_PORT` | The port number for Prometheus metrics. | `9465` | +| `PRIVATE_KEY` | Path to the Solana private key file. | `/path/to/keypair.json` | +| `RATE_LIMIT_CALLS_PER_SECOND` | Maximum number of API calls per second. | `100` | +| `PERP_MARKETS_TO_LOAD` | Number of perpetual markets to load at startup. | `0` | +| `SPOT_MARKETS_TO_LOAD` | Number of spot markets to load at startup. | `5` | +| `ELASTICACHE_HOST` | (for websocket server) Redis host endpoint. | `localhost` | +| `ELASTICACHE_PORT` | (for websocket server) Redis port. | `6379` | +| `REDIS_CLIENT` | (for websocket server) Redis client type (DLOB/DLOB_HELIUS). | `DLOB` | +| `WS_PORT` | (for websocket server) The port to run the websocket server on. | `3000` | Note: multiple Redis hosts can be provided by providing a comma separated string. - ## HTTP mode The HTTP server as documented [here](https://drift-labs.github.io/v2-teacher/?python#orderbook-trades-dlob-server) can be run with, and by default accessible on `http://127.0.0.1:6969`: + ``` yarn run dev ``` @@ -68,22 +99,26 @@ yarn run dev The websocket server has 2 components, the `dlob-publisher` that takes frequent snapshots of the DLOB and publishes them to Redis, and `ws-manager` listens for new connections and sends the latest DLOB to ws clients, the two components communicate through Redis pub-sub. To run the websocket server, a Redis cache is required, and the following environment variables must be set: -* `REDIS_HOSTS` -* `REDIS_PASSWORDS` -* `REDIS_PORTS` + +- `REDIS_HOSTS` +- `REDIS_PASSWORDS` +- `REDIS_PORTS` In the first terminal, start the redis cluster: + ``` bash redisCluster.sh start bash redisCluster.sh create ``` In second terminal, run: + ``` yarn run dlob-publish ``` In a third terminal, run: + ``` yarn run ws-manager ``` @@ -91,6 +126,7 @@ yarn run ws-manager Then connect to the ws server at ws://127.0.0.1:3000 When you're done, stop the redis cluster: + ``` bash redisCluster.sh stop ``` @@ -99,4 +135,33 @@ bash redisCluster.sh stop Documentation for connecting to the dlob server are available [here](https://drift-labs.github.io/v2-teacher/?python#orderbook-trades-dlob-server) -TODO: complete client examples. \ No newline at end of file +TODO: complete client examples. + +## TOB (Top of Book) Monitoring [#tob-monitoring] + +The server includes a TOB monitoring feature that detects when order books become stuck due to ghost/stuck orders. This is particularly useful for gRPC connections that may miss updates. + +### Configuration + +Set the following environment variables to enable and configure TOB monitoring: + +- `ENABLE_TOB_MONITORING=true` - Enable TOB monitoring (default: false) +- `TOB_CHECK_INTERVAL=60000` - How often to check TOB (default: 60 seconds) +- `TOB_STUCK_THRESHOLD=60000` - How long TOB can be stuck before resubscribing (default: 60 seconds) +- `TOP_MONITORING_ENABLED_PERP_MARKETS=0,1,2` - Comma-separated list of perp market indexes to monitor for TOB (default: 0,1,2 for SOL-PERP, BTC-PERP, ETH-PERP) + +### How it works + +1. Checks if the current node is configured to handle any TOB monitoring markets +2. Only enables monitoring if the node has TOB monitoring markets configured +3. Monitors the top bid/ask prices for TOB monitoring perp markets on this node +4. If TOB hasn't changed for the configured threshold time, triggers a resubscribe +5. Performs unsubscribe → subscribe → fetch sequence on the OrderSubscriber instance +6. Logs warnings and updates metrics for monitoring + +### Metrics + +The following metrics are available for TOB monitoring: + +- `tob_resubscribe` - Counter for resubscribe attempts (with success/failure labels) +- `tob_stuck_duration` - Gauge showing how long TOB has been stuck for each market diff --git a/scripts/check-secrets.sh b/scripts/check-secrets.sh new file mode 100755 index 0000000..7386ce6 --- /dev/null +++ b/scripts/check-secrets.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +# Pre-commit hook to check for potential secrets +# This script checks for common patterns that might indicate secrets being committed + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo "🔍 Checking for potential secrets in staged files..." + +# Get list of staged files +STAGED_FILES=$(git diff --cached --name-only --diff-filter=ACM) + +if [ -z "$STAGED_FILES" ]; then + echo "✅ No files staged for commit" + exit 0 +fi + +# Patterns to check for (add more as needed) +PATTERNS=( + # RPC URLs and endpoints + "https://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + "wss://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + "ws://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + "http://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + + # API keys and tokens (common patterns) + "[a-zA-Z0-9]{32,}" # Long alphanumeric strings (likely tokens) + "sk-[a-zA-Z0-9]{20,}" # Stripe-like keys + "pk_[a-zA-Z0-9]{20,}" # Public keys + "sk_[a-zA-Z0-9]{20,}" # Secret keys + + # Private keys and addresses + "[1-9A-HJ-NP-Za-km-z]{32,}" # Base58 encoded strings (likely private keys) + "0x[a-fA-F0-9]{40}" # Ethereum addresses + "[1-9A-HJ-NP-Za-km-z]{44}" # Solana addresses + + # Common secret patterns + "password.*=.*['\"][^'\"]{8,}['\"]" + "secret.*=.*['\"][^'\"]{8,}['\"]" + "token.*=.*['\"][^'\"]{8,}['\"]" + "key.*=.*['\"][^'\"]{8,}['\"]" + "api_key.*=.*['\"][^'\"]{8,}['\"]" + "private_key.*=.*['\"][^'\"]{8,}['\"]" + + # Environment variables that might contain secrets + "ENDPOINT.*=.*['\"][^'\"]{10,}['\"]" + "WS_ENDPOINT.*=.*['\"][^'\"]{10,}['\"]" + "GRPC_ENDPOINT.*=.*['\"][^'\"]{10,}['\"]" + "TOKEN.*=.*['\"][^'\"]{8,}['\"]" + "API_KEY.*=.*['\"][^'\"]{8,}['\"]" + "SECRET.*=.*['\"][^'\"]{8,}['\"]" +) + +# Files to exclude from checking +EXCLUDE_PATTERNS=( + "node_modules" + ".git" + "*.lock" + "*.lockb" + "*.so" + "*.dll" + "*.exe" + "*.bin" + "*.min.js" + "*.min.css" + "*.map" + "*.log" + "*.tmp" + "*.cache" + "build/" + "dist/" + ".env.example" + "scripts/check-secrets.sh" # Exclude this script itself +) + +# Function to check if file should be excluded +should_exclude() { + local file="$1" + for pattern in "${EXCLUDE_PATTERNS[@]}"; do + if [[ "$file" == *"$pattern"* ]] || [[ "$file" =~ $pattern ]]; then + return 0 + fi + done + return 1 +} + +# Function to check for patterns in a file +check_file() { + local file="$1" + local found_secrets=false + + # Skip binary files + if file "$file" | grep -q "binary"; then + return 0 + fi + + # Check each pattern + for pattern in "${PATTERNS[@]}"; do + if git grep --cached "$pattern" "$file" >/dev/null 2>&1; then + echo -e "${RED}⚠️ Potential secret found in $file${NC}" + echo -e "${YELLOW}Pattern: $pattern${NC}" + git grep --cached "$pattern" "$file" | head -3 + echo "" + found_secrets=true + fi + done + + if [ "$found_secrets" = true ]; then + return 1 + fi + + return 0 +} + +# Check each staged file +has_secrets=false +for file in $STAGED_FILES; do + if should_exclude "$file"; then + echo -e "${GREEN}✅ Skipping excluded file: $file${NC}" + continue + fi + + if ! check_file "$file"; then + has_secrets=true + fi +done + +if [ "$has_secrets" = true ]; then + echo -e "${RED}❌ Potential secrets detected!${NC}" + echo -e "${YELLOW}Please review the above matches and ensure no real secrets are being committed.${NC}" + echo -e "${YELLOW}If these are false positives, you can:${NC}" + echo -e "${YELLOW}1. Add the file to .gitignore${NC}" + echo -e "${YELLOW}2. Use placeholder values instead of real secrets${NC}" + echo -e "${YELLOW}3. Add the pattern to the exclude list in this script${NC}" + exit 1 +else + echo -e "${GREEN}✅ No potential secrets detected${NC}" + exit 0 +fi \ No newline at end of file diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index d24f121..118ac40 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -87,6 +87,14 @@ const kinesisRecordsSentCounter = metricsV2.addCounter( 'kinesis_records_sent', 'Number of records sent to Kinesis' ); +const tobResubscribeCounter = metricsV2.addCounter( + 'tob_resubscribe', + 'Number of TOB resubscribes triggered' +); +const tobStuckGauge = metricsV2.addGauge( + 'tob_stuck_duration', + 'Duration TOB has been stuck for each market' +); metricsV2.finalizeObservables(); //@ts-ignore @@ -94,7 +102,7 @@ const sdkConfig = initialize({ env: process.env.ENV }); let driftClient: DriftClient; const opts = program.opts(); -setLogLevel(opts.debug ? 'debug' : 'info'); +setLogLevel('debug'); const useGrpc = process.env.USE_GRPC?.toLowerCase() === 'true'; const useWebsocket = process.env.USE_WEBSOCKET?.toLowerCase() === 'true'; @@ -116,6 +124,16 @@ const WS_FALLBACK_FETCH_INTERVAL = 60_000; const KILLSWITCH_SLOT_DIFF_THRESHOLD = parseInt(process.env.KILLSWITCH_SLOT_DIFF_THRESHOLD) || 200; +// TOB monitoring configuration +const ENABLE_TOB_MONITORING = + process.env.ENABLE_TOB_MONITORING?.toLowerCase() === 'true'; +const TOB_CHECK_INTERVAL = parseInt(process.env.TOB_CHECK_INTERVAL) || 60_000; // 1 minute +const TOB_STUCK_THRESHOLD = parseInt(process.env.TOB_STUCK_THRESHOLD) || 60_000; // 1 minute without change +const TOB_MONITORING_ENABLED_PERP_MARKETS = process.env + .TOB_MONITORING_ENABLED_PERP_MARKETS + ? parsePositiveIntArray(process.env.TOB_MONITORING_ENABLED_PERP_MARKETS) + : [0, 1, 2]; // Default to SOL-PERP, BTC-PERP, ETH-PERP + // comma separated list of perp market indexes to load: i.e. 0,1,2,3 const PERP_MARKETS_TO_LOAD = process.env.PERP_MARKETS_TO_LOAD !== undefined @@ -142,6 +160,16 @@ logger.info( ); logger.info(`DriftEnv: ${driftEnv}`); logger.info(`Commit: ${commitHash}`); +logger.info( + `TOB Monitoring: ${ENABLE_TOB_MONITORING ? 'enabled' : 'disabled'}` +); +if (ENABLE_TOB_MONITORING) { + logger.info(`TOB Check Interval: ${TOB_CHECK_INTERVAL}ms`); + logger.info(`TOB Stuck Threshold: ${TOB_STUCK_THRESHOLD}ms`); + logger.info( + `TOB Monitoring Markets: ${TOB_MONITORING_ENABLED_PERP_MARKETS.join(', ')}` + ); +} let MARKET_SUBSCRIBERS: SubscriberLookup = {}; @@ -437,6 +465,8 @@ const main = async () => { ); let dlobProvider: DLOBProvider; + let orderSubscriber: OrderSubscriberFiltered | undefined; + if (useOrderSubscriber) { let subscriptionConfig: any = { type: 'polling', @@ -480,7 +510,7 @@ const main = async () => { }; } - const orderSubscriber = new OrderSubscriberFiltered({ + orderSubscriber = new OrderSubscriberFiltered({ driftClient, subscriptionConfig, ignoreList, @@ -623,6 +653,158 @@ const main = async () => { }); }, 10_000); + // TOB monitoring for configured perp markets to detect stuck orders + // Check if this node has any TOB monitoring markets configured + const tobMonitoringMarketsInThisNode = + TOB_MONITORING_ENABLED_PERP_MARKETS.filter((marketIndex) => + perpMarketInfos.some((market) => market.marketIndex === marketIndex) + ); + + const shouldEnableTobMonitoring = + ENABLE_TOB_MONITORING && + useOrderSubscriber && + tobMonitoringMarketsInThisNode.length > 0; + + // Track last TOB update times and values for each TOB monitoring market + const lastTobUpdateTimes = new Map(); + const lastTobValues = new Map(); + + // Initialize TOB tracking for TOB monitoring markets in this node + tobMonitoringMarketsInThisNode.forEach((marketIndex) => { + lastTobUpdateTimes.set(marketIndex, Date.now()); + lastTobValues.set(marketIndex, { bid: '', ask: '' }); + }); + + // Log TOB monitoring status + if (ENABLE_TOB_MONITORING) { + logger.info( + `TOB Monitoring Markets in this node: ${tobMonitoringMarketsInThisNode.join( + ', ' + )}` + ); + logger.info( + `TOB Monitoring active: ${shouldEnableTobMonitoring ? 'yes' : 'no'}` + ); + } + + // TOB monitoring function + const checkTobForStuckOrders = async () => { + if (!shouldEnableTobMonitoring) { + return; // Only monitor when using OrderSubscriber, TOB monitoring is enabled, and node has major markets + } + + logger.debug('Starting TOB monitoring check'); + + const currentTime = Date.now(); + + for (const marketIndex of tobMonitoringMarketsInThisNode) { + try { + // Get current TOB from DLOB + const slot = slotSource.getSlot(); + const dlob = await dlobProvider.getDLOB(slot); + + // Get oracle data for the market + const oracleData = driftClient.getOracleDataForPerpMarket(marketIndex); + + // Get L2 orderbook to check TOB + const l2OrderBook = dlob.getL2({ + marketIndex, + marketType: { perp: {} }, + slot, + oraclePriceData: oracleData, + depth: 1, // Only need top level + }); + + const bestBid = l2OrderBook.bids[0]?.price; + const bestAsk = l2OrderBook.asks[0]?.price; + + if (bestBid && bestAsk) { + const currentTobValues = { + bid: bestBid.toString(), + ask: bestAsk.toString(), + }; + const lastTobValue = lastTobValues.get(marketIndex); + const lastUpdate = lastTobUpdateTimes.get(marketIndex); + + // Check if TOB has changed + const tobChanged = + !lastTobValue || + lastTobValue.bid !== currentTobValues.bid || + lastTobValue.ask !== currentTobValues.ask; + + if (tobChanged) { + // TOB changed, update tracking + lastTobValues.set(marketIndex, currentTobValues); + lastTobUpdateTimes.set(marketIndex, currentTime); + logger.debug( + `TOB updated for market ${marketIndex}: bid=${currentTobValues.bid}, ask=${currentTobValues.ask}` + ); + } else if ( + lastUpdate && + currentTime - lastUpdate > TOB_STUCK_THRESHOLD + ) { + // TOB has been stuck for too long, trigger resubscribe + const stuckDuration = (currentTime - lastUpdate) / 1000; + logger.warn( + `TOB stuck for market ${marketIndex} for ${stuckDuration}s, triggering resubscribe` + ); + + // Update metrics + tobStuckGauge.setLatestValue(stuckDuration, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + }); + + // Get the OrderSubscriber instance for resubscribe + if (orderSubscriber) { + try { + // Resubscribe and fetch to clear stuck state + await orderSubscriber.unsubscribe(); + await orderSubscriber.subscribe(); + await orderSubscriber.fetch(); + + logger.info( + `Successfully resubscribed OrderSubscriber for market ${marketIndex}` + ); + + // Update metrics + tobResubscribeCounter.add(1, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + success: 'true', + }); + + // Reset the timer after successful resubscribe + lastTobUpdateTimes.set(marketIndex, currentTime); + } catch (error) { + logger.error( + `Failed to resubscribe OrderSubscriber for market ${marketIndex}:`, + error + ); + + // Update metrics for failed resubscribe + tobResubscribeCounter.add(1, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + success: 'false', + }); + } + } else { + logger.error( + `OrderSubscriber not available for market ${marketIndex}` + ); + } + } + } + } catch (error) { + logger.error(`Error checking TOB for market ${marketIndex}:`, error); + } + } + }; + + // Start TOB monitoring + setInterval(checkTobForStuckOrders, TOB_CHECK_INTERVAL); + const handleStartup = async (_req, res, _next) => { if (driftClient.isSubscribed && dlobProvider.size() > 0) { res.writeHead(200);