From b04762863d87ffe6371bc96330b8879cd6e619df Mon Sep 17 00:00:00 2001 From: Nick Caradonna Date: Tue, 15 Jul 2025 14:47:04 -0400 Subject: [PATCH 1/9] auctionParams endpoint accept full precision amount param --- src/utils/tests/auctionParams.test.ts | 29 ++++++++++++++------------- src/utils/utils.ts | 7 ++----- 2 files changed, 17 insertions(+), 19 deletions(-) diff --git a/src/utils/tests/auctionParams.test.ts b/src/utils/tests/auctionParams.test.ts index ab9ea1d..3eec8ce 100644 --- a/src/utils/tests/auctionParams.test.ts +++ b/src/utils/tests/auctionParams.test.ts @@ -5,7 +5,8 @@ import { MarketType, ZERO, QUOTE_PRECISION, - PRICE_PRECISION + PRICE_PRECISION, + BASE_PRECISION } from '@drift-labs/sdk'; import { createMarketBasedAuctionParams, @@ -191,7 +192,7 @@ describe('Auction Parameters Functions', () => { marketIndex: 0, marketType: 'perp' as any, direction: 'long' as any, - amount: '100', + amount: new BN(100).mul(BASE_PRECISION).toString(), // 100 in BASE_PRECISION assetType: 'base' as any, auctionDuration: 30, auctionStartPriceOffset: 0, @@ -261,6 +262,7 @@ describe('Auction Parameters Functions', () => { it('should handle quote-to-base conversion with realistic prices', async () => { const solPrice = 160; // $160 SOL price const quoteAmount = 1000; // $1,000 worth + const quoteAmountInPrecision = new BN(quoteAmount).mul(QUOTE_PRECISION); // Convert to quote precision mockDriftClient.getOracleDataForPerpMarket.mockReturnValue({ price: new BN(solPrice).mul(PRICE_PRECISION), @@ -282,7 +284,7 @@ describe('Auction Parameters Functions', () => { const quoteParams = { ...validParams, - amount: quoteAmount.toString(), + amount: quoteAmountInPrecision.toString(), assetType: 'quote' as any, }; @@ -300,10 +302,9 @@ describe('Auction Parameters Functions', () => { expect(baseAmount).toBeDefined(); expect(baseAmount.gt(ZERO)).toBe(true); - // Calculate expected base amount: (quoteAmount * QUOTE_PRECISION * BASE_PRECISION) / entryPrice - const expectedBaseAmount = new BN(quoteAmount) - .mul(QUOTE_PRECISION) - .mul(new BN(10).pow(new BN(9))) // BASE_PRECISION = 10^9 + // Calculate expected base amount: (amount * BASE_PRECISION) / entryPrice + const expectedBaseAmount = quoteAmountInPrecision + .mul(BASE_PRECISION) .div(result.data?.estimatedPrices.entryPrice); expect(baseAmount.toString()).toBe(expectedBaseAmount.toString()); @@ -414,7 +415,7 @@ describe('Auction Parameters Functions', () => { // Test small order const smallOrderParams = { ...validParams, - amount: '10', // Small order + amount: new BN(10).mul(BASE_PRECISION).toString(), // 10 in BASE_PRECISION }; const smallResult = await mapToMarketOrderParams( @@ -430,7 +431,7 @@ describe('Auction Parameters Functions', () => { // Test large order const largeOrderParams = { ...validParams, - amount: '1000', // Large order + amount: new BN(1000).mul(BASE_PRECISION).toString(), // 1000 in BASE_PRECISION }; const largeResult = await mapToMarketOrderParams( @@ -565,7 +566,7 @@ describe('Auction Parameters Functions', () => { marketIndex: 0, marketType: 'perp' as any, direction: 'long' as any, - amount: '100', + amount: new BN(100).mul(BASE_PRECISION).toString(), // 100 in BASE_PRECISION assetType: 'base' as any, auctionStartPriceOffsetFrom: 'marketBased' as any, auctionStartPriceOffset: 'marketBased' as any, @@ -578,7 +579,7 @@ describe('Auction Parameters Functions', () => { marketIndex: 5, marketType: 'perp' as any, direction: 'short' as any, - amount: '500', + amount: new BN(500).mul(QUOTE_PRECISION).toString(), // 500 in QUOTE_PRECISION assetType: 'quote' as any, auctionStartPriceOffsetFrom: 'marketBased' as any, auctionStartPriceOffset: 'marketBased' as any, @@ -591,7 +592,7 @@ describe('Auction Parameters Functions', () => { marketIndex: 2, marketType: 'spot' as any, direction: 'long' as any, - amount: '1000', + amount: new BN(1000).mul(BASE_PRECISION).toString(), // 1000 in BASE_PRECISION assetType: 'base' as any, auctionStartPriceOffsetFrom: 'marketBased' as any, auctionStartPriceOffset: 'marketBased' as any, @@ -621,7 +622,7 @@ describe('Auction Parameters Functions', () => { marketIndex: 0, // Major market - would normally get 'mark' and 0 marketType: 'perp' as any, direction: 'long' as any, - amount: '100', + amount: new BN(100).mul(BASE_PRECISION).toString(), // 100 in BASE_PRECISION assetType: 'base' as any, auctionStartPriceOffsetFrom: 'oracle' as any, // explicit value auctionStartPriceOffset: 0.25, // explicit value @@ -644,7 +645,7 @@ describe('Auction Parameters Functions', () => { marketIndex: 1, marketType: 'perp' as any, direction: 'long' as any, - amount: '100', + amount: new BN(100).mul(BASE_PRECISION).toString(), // 100 in BASE_PRECISION assetType: 'base' as any, auctionStartPriceOffsetFrom: 'marketBased' as any, auctionStartPriceOffset: 'marketBased' as any, diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 612f1a0..553c23f 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -889,11 +889,8 @@ export const mapToMarketOrderParams = async ( ? PositionDirection.LONG : PositionDirection.SHORT; - // Convert amount string to BN based on assetType - const amount = - params.assetType === 'base' - ? stringToBN(params.amount).mul(BASE_PRECISION) - : stringToBN(params.amount).mul(QUOTE_PRECISION); + // Convert amount string to BN - amount is already in base or quote precision + const amount = stringToBN(params.amount); // Convert additionalEndPriceBuffer string to BN with PRICE_PRECISION (1e6) if provided const additionalEndPriceBuffer = params.additionalEndPriceBuffer From 9eb677112376dc35d49c1c7da641397b04e75132 Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Tue, 29 Jul 2025 13:05:21 -0600 Subject: [PATCH 2/9] feat: bump drift common --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 55975ac..b2751c8 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 55975ac61a2369c201b3b8cbe046b9a101de75e3 +Subproject commit b2751c84a158e82420ac3709292cd9b4dd485687 From b987718f57b356fa250a5b8820eff6724f0de44c Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Tue, 29 Jul 2025 15:39:02 -0600 Subject: [PATCH 3/9] feat: use new ws v2 subscriber --- src/publishers/dlobPublisher.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index aead36e..7416f71 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -9,7 +9,6 @@ import { UserMap, Wallet, BulkAccountLoader, - OrderSubscriber, SlotSource, DriftClientSubscriptionConfig, SlotSubscriber, @@ -18,10 +17,9 @@ import { PerpMarketConfig, SpotMarketConfig, PhoenixSubscriber, - MarketType, - OraclePriceData, - ONE, decodeName, + WebSocketAccountSubscriberV2, + ONE, } from '@drift-labs/sdk'; import { RedisClient, RedisClientPrefix } from '@drift/common/clients'; @@ -29,7 +27,6 @@ import { logger, setLogLevel } from '../utils/logger'; import { SubscriberLookup, getOpenbookSubscriber, - l2WithBNToStrings, parsePositiveIntArray, sleep, } from '../utils/utils'; @@ -391,6 +388,7 @@ const main = async () => { commitment: stateCommitment, resubTimeoutMs: 30_000, logResubMessages: true, + accountSubscriber: WebSocketAccountSubscriberV2, }; slotSubscriber = new SlotSubscriber(connection, { resubTimeoutMs: 10_000, From faddfbb8ecc170abe282fbd792d003103c55163a Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Wed, 30 Jul 2025 14:17:49 -0600 Subject: [PATCH 4/9] chore: bump common --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index b2751c8..8f94c37 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit b2751c84a158e82420ac3709292cd9b4dd485687 +Subproject commit 8f94c374ad4de9e81b2e8ca197945b1ab36e7a2c From ff7959f14d43202ee1495394af4a34c57bbb8234 Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Wed, 30 Jul 2025 22:48:25 -0600 Subject: [PATCH 5/9] fix: perp market acct subscriber prop wrong name drift client --- src/publishers/dlobPublisher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 7416f71..b52297f 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -388,7 +388,7 @@ const main = async () => { commitment: stateCommitment, resubTimeoutMs: 30_000, logResubMessages: true, - accountSubscriber: WebSocketAccountSubscriberV2, + perpMarketAccountSubscriber: WebSocketAccountSubscriberV2, }; slotSubscriber = new SlotSubscriber(connection, { resubTimeoutMs: 10_000, From 2b0b3a534e1ac784a8d73bff0420dac4f263bc2e Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Thu, 31 Jul 2025 17:24:30 -0600 Subject: [PATCH 6/9] chore: bump common --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 8f94c37..6721454 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 8f94c374ad4de9e81b2e8ca197945b1ab36e7a2c +Subproject commit 67214545014ef420d80b5d6fed8e11741311c54c From 2680fef18dd578720459da297e678f73377b8df7 Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Fri, 1 Aug 2025 14:55:53 -0600 Subject: [PATCH 7/9] feat: TOB monitoring logic --- .husky/pre-commit | 22 ++++ README.md | 115 +++++++++++++++----- scripts/check-secrets.sh | 145 +++++++++++++++++++++++++ src/publishers/dlobPublisher.ts | 186 +++++++++++++++++++++++++++++++- 4 files changed, 441 insertions(+), 27 deletions(-) create mode 100755 .husky/pre-commit create mode 100755 scripts/check-secrets.sh diff --git a/.husky/pre-commit b/.husky/pre-commit new file mode 100755 index 0000000..fe7ea7b --- /dev/null +++ b/.husky/pre-commit @@ -0,0 +1,22 @@ +#!/usr/bin/env sh +. "$(dirname -- "$0")/_/husky.sh" + +# Pre-commit hook to check for potential secrets +echo "🔍 Running secret check..." + +# Run the secret checking script +if [ -f "scripts/check-secrets.sh" ]; then + ./scripts/check-secrets.sh + if [ $? -ne 0 ]; then + echo "❌ Pre-commit hook failed. Please fix the issues above before committing." + echo "💡 If these are false positives, you can:" + echo " 1. Add the file to .gitignore" + echo " 2. Use placeholder values instead of real secrets" + echo " 3. Bypass the hook with: git commit --no-verify" + exit 1 + fi +else + echo "⚠️ Warning: check-secrets.sh not found" +fi + +echo "✅ Pre-commit checks passed" \ No newline at end of file diff --git a/README.md b/README.md index aeb0aac..4066f58 100644 --- a/README.md +++ b/README.md @@ -14,51 +14,82 @@ This is the backend server that provides a REST API for the drift [DLOB](https://docs.drift.trade/about-v2/decentralized-orderbook). +## Features + +- Real-time DLOB data publishing +- Support for both perp and spot markets +- Multiple subscription methods (WebSocket, gRPC, polling) +- Health checks and metrics +- **TOB (Top of Book) monitoring for stuck orders** (see [TOB Monitoring Configuration](#tob-monitoring-configuration)) + # Run the server ## Setup The build dependencies + ``` git submodule update --init bash build_all.sh ``` First set the necessary environment variables: + ``` cp .env.example .env ``` -## Environment Configuration +## Security + +### Pre-commit Hook + +This repository uses [Husky](https://typicode.github.io/husky/) to manage Git hooks. A pre-commit hook automatically checks for potential secrets before each commit. The hook will: + +- Scan for RPC URLs, API keys, tokens, and other potential secrets +- Prevent commits that contain suspicious patterns +- Provide helpful guidance if false positives are detected + +The hook is automatically installed when you run `npm install` (via the `prepare` script). If you need to bypass it for a specific commit, use: + +```bash +git commit --no-verify +``` + +### Scripts + +The `scripts/` directory contains utility scripts: + +- `check-secrets.sh` - Secret detection script used by the pre-commit hook + +## Environment Variables To properly configure the DLOB server, set the following environment variables in your `.env` file: -| Variable | Description | Example Value | -|---------------------------|-----------------------------------------------------------|--------------------------------------------------------------------| -| `ENDPOINT` | The Solana RPC node http endpoint. | `https://your-private-rpc-node.com` | -| `WS_ENDPOINT` | The Solana RPC node websocket endpoint. | `wss://your-private-rpc-node.com` | -| `USE_WEBSOCKET` | Flag to enable WebSocket connection. | `true` | -| `USE_ORDER_SUBSCRIBER` | Flag to enable order subscriber DLOB source. | `true` | -| `DISABLE_GPA_REFRESH` | Flag to disable periodic refresh using `getProgramAccounts`. | `true` | -| `ENV` | The network environment the server is connecting to. | `mainnet-beta` | -| `PORT` | The port number the HTTP server listens on. | `6969` | -| `METRICS_PORT` | The port number for Prometheus metrics. | `9465` | -| `PRIVATE_KEY` | Path to the Solana private key file. | `/path/to/keypair.json` | -| `RATE_LIMIT_CALLS_PER_SECOND` | Maximum number of API calls per second. | `100` | -| `PERP_MARKETS_TO_LOAD` | Number of perpetual markets to load at startup. | `0` | -| `SPOT_MARKETS_TO_LOAD` | Number of spot markets to load at startup. | `5` | -| `ELASTICACHE_HOST` | (for websocket server) Redis host endpoint. | `localhost` | -| `ELASTICACHE_PORT` | (for websocket server) Redis port. | `6379` | -| `REDIS_CLIENT` | (for websocket server) Redis client type (DLOB/DLOB_HELIUS).| `DLOB` | -| `WS_PORT` | (for websocket server) The port to run the websocket server on.| `3000` | - +| Variable | Description | Example Value | +| ----------------------------- | --------------------------------------------------------------- | ----------------------------------- | +| `ENDPOINT` | The Solana RPC node http endpoint. | `https://your-private-rpc-node.com` | +| `WS_ENDPOINT` | The Solana RPC node websocket endpoint. | `wss://your-private-rpc-node.com` | +| `USE_WEBSOCKET` | Flag to enable WebSocket connection. | `true` | +| `USE_ORDER_SUBSCRIBER` | Flag to enable order subscriber DLOB source. | `true` | +| `DISABLE_GPA_REFRESH` | Flag to disable periodic refresh using `getProgramAccounts`. | `true` | +| `ENV` | The network environment the server is connecting to. | `mainnet-beta` | +| `PORT` | The port number the HTTP server listens on. | `6969` | +| `METRICS_PORT` | The port number for Prometheus metrics. | `9465` | +| `PRIVATE_KEY` | Path to the Solana private key file. | `/path/to/keypair.json` | +| `RATE_LIMIT_CALLS_PER_SECOND` | Maximum number of API calls per second. | `100` | +| `PERP_MARKETS_TO_LOAD` | Number of perpetual markets to load at startup. | `0` | +| `SPOT_MARKETS_TO_LOAD` | Number of spot markets to load at startup. | `5` | +| `ELASTICACHE_HOST` | (for websocket server) Redis host endpoint. | `localhost` | +| `ELASTICACHE_PORT` | (for websocket server) Redis port. | `6379` | +| `REDIS_CLIENT` | (for websocket server) Redis client type (DLOB/DLOB_HELIUS). | `DLOB` | +| `WS_PORT` | (for websocket server) The port to run the websocket server on. | `3000` | Note: multiple Redis hosts can be provided by providing a comma separated string. - ## HTTP mode The HTTP server as documented [here](https://drift-labs.github.io/v2-teacher/?python#orderbook-trades-dlob-server) can be run with, and by default accessible on `http://127.0.0.1:6969`: + ``` yarn run dev ``` @@ -68,22 +99,26 @@ yarn run dev The websocket server has 2 components, the `dlob-publisher` that takes frequent snapshots of the DLOB and publishes them to Redis, and `ws-manager` listens for new connections and sends the latest DLOB to ws clients, the two components communicate through Redis pub-sub. To run the websocket server, a Redis cache is required, and the following environment variables must be set: -* `REDIS_HOSTS` -* `REDIS_PASSWORDS` -* `REDIS_PORTS` + +- `REDIS_HOSTS` +- `REDIS_PASSWORDS` +- `REDIS_PORTS` In the first terminal, start the redis cluster: + ``` bash redisCluster.sh start bash redisCluster.sh create ``` In second terminal, run: + ``` yarn run dlob-publish ``` In a third terminal, run: + ``` yarn run ws-manager ``` @@ -91,6 +126,7 @@ yarn run ws-manager Then connect to the ws server at ws://127.0.0.1:3000 When you're done, stop the redis cluster: + ``` bash redisCluster.sh stop ``` @@ -99,4 +135,33 @@ bash redisCluster.sh stop Documentation for connecting to the dlob server are available [here](https://drift-labs.github.io/v2-teacher/?python#orderbook-trades-dlob-server) -TODO: complete client examples. \ No newline at end of file +TODO: complete client examples. + +## TOB (Top of Book) Monitoring [#tob-monitoring] + +The server includes a TOB monitoring feature that detects when order books become stuck due to ghost/stuck orders. This is particularly useful for gRPC connections that may miss updates. + +### Configuration + +Set the following environment variables to enable and configure TOB monitoring: + +- `ENABLE_TOB_MONITORING=true` - Enable TOB monitoring (default: false) +- `TOB_CHECK_INTERVAL=60000` - How often to check TOB (default: 60 seconds) +- `TOB_STUCK_THRESHOLD=60000` - How long TOB can be stuck before resubscribing (default: 60 seconds) +- `TOP_MONITORING_ENABLED_PERP_MARKETS=0,1,2` - Comma-separated list of perp market indexes to monitor for TOB (default: 0,1,2 for SOL-PERP, BTC-PERP, ETH-PERP) + +### How it works + +1. Checks if the current node is configured to handle any TOB monitoring markets +2. Only enables monitoring if the node has TOB monitoring markets configured +3. Monitors the top bid/ask prices for TOB monitoring perp markets on this node +4. If TOB hasn't changed for the configured threshold time, triggers a resubscribe +5. Performs unsubscribe → subscribe → fetch sequence on the OrderSubscriber instance +6. Logs warnings and updates metrics for monitoring + +### Metrics + +The following metrics are available for TOB monitoring: + +- `tob_resubscribe` - Counter for resubscribe attempts (with success/failure labels) +- `tob_stuck_duration` - Gauge showing how long TOB has been stuck for each market diff --git a/scripts/check-secrets.sh b/scripts/check-secrets.sh new file mode 100755 index 0000000..7386ce6 --- /dev/null +++ b/scripts/check-secrets.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +# Pre-commit hook to check for potential secrets +# This script checks for common patterns that might indicate secrets being committed + +set -e + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo "🔍 Checking for potential secrets in staged files..." + +# Get list of staged files +STAGED_FILES=$(git diff --cached --name-only --diff-filter=ACM) + +if [ -z "$STAGED_FILES" ]; then + echo "✅ No files staged for commit" + exit 0 +fi + +# Patterns to check for (add more as needed) +PATTERNS=( + # RPC URLs and endpoints + "https://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + "wss://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + "ws://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + "http://[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" + + # API keys and tokens (common patterns) + "[a-zA-Z0-9]{32,}" # Long alphanumeric strings (likely tokens) + "sk-[a-zA-Z0-9]{20,}" # Stripe-like keys + "pk_[a-zA-Z0-9]{20,}" # Public keys + "sk_[a-zA-Z0-9]{20,}" # Secret keys + + # Private keys and addresses + "[1-9A-HJ-NP-Za-km-z]{32,}" # Base58 encoded strings (likely private keys) + "0x[a-fA-F0-9]{40}" # Ethereum addresses + "[1-9A-HJ-NP-Za-km-z]{44}" # Solana addresses + + # Common secret patterns + "password.*=.*['\"][^'\"]{8,}['\"]" + "secret.*=.*['\"][^'\"]{8,}['\"]" + "token.*=.*['\"][^'\"]{8,}['\"]" + "key.*=.*['\"][^'\"]{8,}['\"]" + "api_key.*=.*['\"][^'\"]{8,}['\"]" + "private_key.*=.*['\"][^'\"]{8,}['\"]" + + # Environment variables that might contain secrets + "ENDPOINT.*=.*['\"][^'\"]{10,}['\"]" + "WS_ENDPOINT.*=.*['\"][^'\"]{10,}['\"]" + "GRPC_ENDPOINT.*=.*['\"][^'\"]{10,}['\"]" + "TOKEN.*=.*['\"][^'\"]{8,}['\"]" + "API_KEY.*=.*['\"][^'\"]{8,}['\"]" + "SECRET.*=.*['\"][^'\"]{8,}['\"]" +) + +# Files to exclude from checking +EXCLUDE_PATTERNS=( + "node_modules" + ".git" + "*.lock" + "*.lockb" + "*.so" + "*.dll" + "*.exe" + "*.bin" + "*.min.js" + "*.min.css" + "*.map" + "*.log" + "*.tmp" + "*.cache" + "build/" + "dist/" + ".env.example" + "scripts/check-secrets.sh" # Exclude this script itself +) + +# Function to check if file should be excluded +should_exclude() { + local file="$1" + for pattern in "${EXCLUDE_PATTERNS[@]}"; do + if [[ "$file" == *"$pattern"* ]] || [[ "$file" =~ $pattern ]]; then + return 0 + fi + done + return 1 +} + +# Function to check for patterns in a file +check_file() { + local file="$1" + local found_secrets=false + + # Skip binary files + if file "$file" | grep -q "binary"; then + return 0 + fi + + # Check each pattern + for pattern in "${PATTERNS[@]}"; do + if git grep --cached "$pattern" "$file" >/dev/null 2>&1; then + echo -e "${RED}⚠️ Potential secret found in $file${NC}" + echo -e "${YELLOW}Pattern: $pattern${NC}" + git grep --cached "$pattern" "$file" | head -3 + echo "" + found_secrets=true + fi + done + + if [ "$found_secrets" = true ]; then + return 1 + fi + + return 0 +} + +# Check each staged file +has_secrets=false +for file in $STAGED_FILES; do + if should_exclude "$file"; then + echo -e "${GREEN}✅ Skipping excluded file: $file${NC}" + continue + fi + + if ! check_file "$file"; then + has_secrets=true + fi +done + +if [ "$has_secrets" = true ]; then + echo -e "${RED}❌ Potential secrets detected!${NC}" + echo -e "${YELLOW}Please review the above matches and ensure no real secrets are being committed.${NC}" + echo -e "${YELLOW}If these are false positives, you can:${NC}" + echo -e "${YELLOW}1. Add the file to .gitignore${NC}" + echo -e "${YELLOW}2. Use placeholder values instead of real secrets${NC}" + echo -e "${YELLOW}3. Add the pattern to the exclude list in this script${NC}" + exit 1 +else + echo -e "${GREEN}✅ No potential secrets detected${NC}" + exit 0 +fi \ No newline at end of file diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index b52297f..053f786 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -87,6 +87,14 @@ const kinesisRecordsSentCounter = metricsV2.addCounter( 'kinesis_records_sent', 'Number of records sent to Kinesis' ); +const tobResubscribeCounter = metricsV2.addCounter( + 'tob_resubscribe', + 'Number of TOB resubscribes triggered' +); +const tobStuckGauge = metricsV2.addGauge( + 'tob_stuck_duration', + 'Duration TOB has been stuck for each market' +); metricsV2.finalizeObservables(); //@ts-ignore @@ -94,7 +102,7 @@ const sdkConfig = initialize({ env: process.env.ENV }); let driftClient: DriftClient; const opts = program.opts(); -setLogLevel(opts.debug ? 'debug' : 'info'); +setLogLevel('debug'); const useGrpc = process.env.USE_GRPC?.toLowerCase() === 'true'; const useWebsocket = process.env.USE_WEBSOCKET?.toLowerCase() === 'true'; @@ -116,6 +124,16 @@ const WS_FALLBACK_FETCH_INTERVAL = 60_000; const KILLSWITCH_SLOT_DIFF_THRESHOLD = parseInt(process.env.KILLSWITCH_SLOT_DIFF_THRESHOLD) || 200; +// TOB monitoring configuration +const ENABLE_TOB_MONITORING = + process.env.ENABLE_TOB_MONITORING?.toLowerCase() === 'true'; +const TOB_CHECK_INTERVAL = parseInt(process.env.TOB_CHECK_INTERVAL) || 60_000; // 1 minute +const TOB_STUCK_THRESHOLD = parseInt(process.env.TOB_STUCK_THRESHOLD) || 60_000; // 1 minute without change +const TOB_MONITORING_ENABLED_PERP_MARKETS = process.env + .TOB_MONITORING_ENABLED_PERP_MARKETS + ? parsePositiveIntArray(process.env.TOB_MONITORING_ENABLED_PERP_MARKETS) + : [0, 1, 2]; // Default to SOL-PERP, BTC-PERP, ETH-PERP + // comma separated list of perp market indexes to load: i.e. 0,1,2,3 const PERP_MARKETS_TO_LOAD = process.env.PERP_MARKETS_TO_LOAD !== undefined @@ -142,6 +160,16 @@ logger.info( ); logger.info(`DriftEnv: ${driftEnv}`); logger.info(`Commit: ${commitHash}`); +logger.info( + `TOB Monitoring: ${ENABLE_TOB_MONITORING ? 'enabled' : 'disabled'}` +); +if (ENABLE_TOB_MONITORING) { + logger.info(`TOB Check Interval: ${TOB_CHECK_INTERVAL}ms`); + logger.info(`TOB Stuck Threshold: ${TOB_STUCK_THRESHOLD}ms`); + logger.info( + `TOB Monitoring Markets: ${TOB_MONITORING_ENABLED_PERP_MARKETS.join(', ')}` + ); +} let MARKET_SUBSCRIBERS: SubscriberLookup = {}; @@ -437,6 +465,8 @@ const main = async () => { ); let dlobProvider: DLOBProvider; + let orderSubscriber: OrderSubscriberFiltered | undefined; + if (useOrderSubscriber) { let subscriptionConfig: any = { type: 'polling', @@ -480,7 +510,7 @@ const main = async () => { }; } - const orderSubscriber = new OrderSubscriberFiltered({ + orderSubscriber = new OrderSubscriberFiltered({ driftClient, subscriptionConfig, ignoreList, @@ -623,6 +653,158 @@ const main = async () => { }); }, 10_000); + // TOB monitoring for configured perp markets to detect stuck orders + // Check if this node has any TOB monitoring markets configured + const tobMonitoringMarketsInThisNode = + TOB_MONITORING_ENABLED_PERP_MARKETS.filter((marketIndex) => + perpMarketInfos.some((market) => market.marketIndex === marketIndex) + ); + + const shouldEnableTobMonitoring = + ENABLE_TOB_MONITORING && + useOrderSubscriber && + tobMonitoringMarketsInThisNode.length > 0; + + // Track last TOB update times and values for each TOB monitoring market + const lastTobUpdateTimes = new Map(); + const lastTobValues = new Map(); + + // Initialize TOB tracking for TOB monitoring markets in this node + tobMonitoringMarketsInThisNode.forEach((marketIndex) => { + lastTobUpdateTimes.set(marketIndex, Date.now()); + lastTobValues.set(marketIndex, { bid: '', ask: '' }); + }); + + // Log TOB monitoring status + if (ENABLE_TOB_MONITORING) { + logger.info( + `TOB Monitoring Markets in this node: ${tobMonitoringMarketsInThisNode.join( + ', ' + )}` + ); + logger.info( + `TOB Monitoring active: ${shouldEnableTobMonitoring ? 'yes' : 'no'}` + ); + } + + // TOB monitoring function + const checkTobForStuckOrders = async () => { + if (!shouldEnableTobMonitoring) { + return; // Only monitor when using OrderSubscriber, TOB monitoring is enabled, and node has major markets + } + + logger.debug('Starting TOB monitoring check'); + + const currentTime = Date.now(); + + for (const marketIndex of tobMonitoringMarketsInThisNode) { + try { + // Get current TOB from DLOB + const slot = slotSource.getSlot(); + const dlob = await dlobProvider.getDLOB(slot); + + // Get oracle data for the market + const oracleData = driftClient.getOracleDataForPerpMarket(marketIndex); + + // Get L2 orderbook to check TOB + const l2OrderBook = dlob.getL2({ + marketIndex, + marketType: { perp: {} }, + slot, + oraclePriceData: oracleData, + depth: 1, // Only need top level + }); + + const bestBid = l2OrderBook.bids[0]?.price; + const bestAsk = l2OrderBook.asks[0]?.price; + + if (bestBid && bestAsk) { + const currentTobValues = { + bid: bestBid.toString(), + ask: bestAsk.toString(), + }; + const lastTobValue = lastTobValues.get(marketIndex); + const lastUpdate = lastTobUpdateTimes.get(marketIndex); + + // Check if TOB has changed + const tobChanged = + !lastTobValue || + lastTobValue.bid !== currentTobValues.bid || + lastTobValue.ask !== currentTobValues.ask; + + if (tobChanged) { + // TOB changed, update tracking + lastTobValues.set(marketIndex, currentTobValues); + lastTobUpdateTimes.set(marketIndex, currentTime); + logger.debug( + `TOB updated for market ${marketIndex}: bid=${currentTobValues.bid}, ask=${currentTobValues.ask}` + ); + } else if ( + lastUpdate && + currentTime - lastUpdate > TOB_STUCK_THRESHOLD + ) { + // TOB has been stuck for too long, trigger resubscribe + const stuckDuration = (currentTime - lastUpdate) / 1000; + logger.warn( + `TOB stuck for market ${marketIndex} for ${stuckDuration}s, triggering resubscribe` + ); + + // Update metrics + tobStuckGauge.setLatestValue(stuckDuration, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + }); + + // Get the OrderSubscriber instance for resubscribe + if (orderSubscriber) { + try { + // Resubscribe and fetch to clear stuck state + await orderSubscriber.unsubscribe(); + await orderSubscriber.subscribe(); + await orderSubscriber.fetch(); + + logger.info( + `Successfully resubscribed OrderSubscriber for market ${marketIndex}` + ); + + // Update metrics + tobResubscribeCounter.add(1, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + success: 'true', + }); + + // Reset the timer after successful resubscribe + lastTobUpdateTimes.set(marketIndex, currentTime); + } catch (error) { + logger.error( + `Failed to resubscribe OrderSubscriber for market ${marketIndex}:`, + error + ); + + // Update metrics for failed resubscribe + tobResubscribeCounter.add(1, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + success: 'false', + }); + } + } else { + logger.error( + `OrderSubscriber not available for market ${marketIndex}` + ); + } + } + } + } catch (error) { + logger.error(`Error checking TOB for market ${marketIndex}:`, error); + } + } + }; + + // Start TOB monitoring + setInterval(checkTobForStuckOrders, TOB_CHECK_INTERVAL); + const handleStartup = async (_req, res, _next) => { if (driftClient.isSubscribed && dlobProvider.size() > 0) { res.writeHead(200); From cb29896ce5ede42416dc4fd31b3add305560acb8 Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Fri, 1 Aug 2025 16:00:23 -0600 Subject: [PATCH 8/9] refactor: TOB check uses user+orderId instead of prices --- .vscode/launch.json | 28 ++++++++++++++++++ src/publishers/dlobPublisher.ts | 50 ++++++++++++++++++++------------- 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index 3cce174..2c2d82b 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -11,6 +11,34 @@ "port": 2230, "timeout": 3000, "restart": true + }, + { + "type": "node", + "request": "launch", + "name": "Launch DLOB Publisher", + "runtimeExecutable": "ts-node", + "program": "${workspaceFolder}/src/publishers/dlobPublisher.ts", + "args": ["--D"], + "env": { + "KILLSWITCH_SLOT_DIFF_THRESHOLD": "2500", + "PERP_MARKETS_TO_LOAD": "1,2", + "LOCAL_CACHE": "true", + "RUNNING_LOCAL": "true", + "USE_REDIS": "true", + "ENDPOINT": "", + "WS_ENDPOINT": "", + "USE_WEBSOCKET": "true", + "USE_GRPC": "true", + "GRPC_ENDPOINT": "", + "TOKEN": "", + "USE_ORDER_SUBSCRIBER": "true", + "STATE_COMMITMENT": "confirmed", + "DRIFT_ENV": "mainnet-beta", + "ENV": "mainnet-beta", + "ENABLE_TOB_MONITORING": "true", + "TOB_MONITORING_ENABLED_PERP_MARKETS": "1,2" + }, + "console": "integratedTerminal" } ] } diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 053f786..86b3c8b 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -665,14 +665,17 @@ const main = async () => { useOrderSubscriber && tobMonitoringMarketsInThisNode.length > 0; - // Track last TOB update times and values for each TOB monitoring market + // Track last TOB update times and order IDs for each TOB monitoring market const lastTobUpdateTimes = new Map(); - const lastTobValues = new Map(); + const lastTobOrderIds = new Map< + number, + { bidOrderId: string; askOrderId: string } + >(); // Initialize TOB tracking for TOB monitoring markets in this node tobMonitoringMarketsInThisNode.forEach((marketIndex) => { lastTobUpdateTimes.set(marketIndex, Date.now()); - lastTobValues.set(marketIndex, { bid: '', ask: '' }); + lastTobOrderIds.set(marketIndex, { bidOrderId: '', askOrderId: '' }); }); // Log TOB monitoring status @@ -706,38 +709,45 @@ const main = async () => { // Get oracle data for the market const oracleData = driftClient.getOracleDataForPerpMarket(marketIndex); - // Get L2 orderbook to check TOB - const l2OrderBook = dlob.getL2({ + // Get L3 orderbook to check TOB + const l3OrderBook = dlob.getL3({ marketIndex, marketType: { perp: {} }, slot, oraclePriceData: oracleData, - depth: 1, // Only need top level }); - const bestBid = l2OrderBook.bids[0]?.price; - const bestAsk = l2OrderBook.asks[0]?.price; + const bestBidOrder = l3OrderBook.bids[0]; + const bestAskOrder = l3OrderBook.asks[0]; - if (bestBid && bestAsk) { - const currentTobValues = { - bid: bestBid.toString(), - ask: bestAsk.toString(), + if (bestBidOrder && bestAskOrder) { + // Create order ID strings using maker (user account) and orderId + const currentBidOrderId = `${bestBidOrder.maker.toBase58()}-${ + bestBidOrder.orderId + }`; + const currentAskOrderId = `${bestAskOrder.maker.toBase58()}-${ + bestAskOrder.orderId + }`; + + const currentTobOrderIds = { + bidOrderId: currentBidOrderId, + askOrderId: currentAskOrderId, }; - const lastTobValue = lastTobValues.get(marketIndex); + const lastTobOrderId = lastTobOrderIds.get(marketIndex); const lastUpdate = lastTobUpdateTimes.get(marketIndex); - // Check if TOB has changed + // Check if TOB orders have changed (not just prices) const tobChanged = - !lastTobValue || - lastTobValue.bid !== currentTobValues.bid || - lastTobValue.ask !== currentTobValues.ask; + !lastTobOrderId || + lastTobOrderId.bidOrderId !== currentTobOrderIds.bidOrderId || + lastTobOrderId.askOrderId !== currentTobOrderIds.askOrderId; if (tobChanged) { - // TOB changed, update tracking - lastTobValues.set(marketIndex, currentTobValues); + // TOB orders changed, update tracking + lastTobOrderIds.set(marketIndex, currentTobOrderIds); lastTobUpdateTimes.set(marketIndex, currentTime); logger.debug( - `TOB updated for market ${marketIndex}: bid=${currentTobValues.bid}, ask=${currentTobValues.ask}` + `TOB orders updated for market ${marketIndex}: bidOrderId=${currentTobOrderIds.bidOrderId}, askOrderId=${currentTobOrderIds.askOrderId}` ); } else if ( lastUpdate && From c52c963404b6dd3a6cb69c4a14ec5b1015db2fcb Mon Sep 17 00:00:00 2001 From: Lukas deConantsesznak Date: Sun, 3 Aug 2025 10:40:50 -0600 Subject: [PATCH 9/9] feat: handle case where TOB is empty for bids or asks for TOB checker --- src/publishers/dlobPublisher.ts | 152 ++++++++++++++++---------------- 1 file changed, 76 insertions(+), 76 deletions(-) diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 86b3c8b..476e6e5 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -720,90 +720,90 @@ const main = async () => { const bestBidOrder = l3OrderBook.bids[0]; const bestAskOrder = l3OrderBook.asks[0]; - if (bestBidOrder && bestAskOrder) { - // Create order ID strings using maker (user account) and orderId - const currentBidOrderId = `${bestBidOrder.maker.toBase58()}-${ - bestBidOrder.orderId - }`; - const currentAskOrderId = `${bestAskOrder.maker.toBase58()}-${ - bestAskOrder.orderId - }`; + // Track each side independently, even if one side is empty + const currentBidOrderId = bestBidOrder + ? `${bestBidOrder.maker.toBase58()}-${bestBidOrder.orderId}` + : ''; + const currentAskOrderId = bestAskOrder + ? `${bestAskOrder.maker.toBase58()}-${bestAskOrder.orderId}` + : ''; - const currentTobOrderIds = { - bidOrderId: currentBidOrderId, - askOrderId: currentAskOrderId, - }; - const lastTobOrderId = lastTobOrderIds.get(marketIndex); - const lastUpdate = lastTobUpdateTimes.get(marketIndex); + const currentTobOrderIds = { + bidOrderId: currentBidOrderId, + askOrderId: currentAskOrderId, + }; + const lastTobOrderId = lastTobOrderIds.get(marketIndex); + const lastUpdate = lastTobUpdateTimes.get(marketIndex); - // Check if TOB orders have changed (not just prices) - const tobChanged = - !lastTobOrderId || - lastTobOrderId.bidOrderId !== currentTobOrderIds.bidOrderId || - lastTobOrderId.askOrderId !== currentTobOrderIds.askOrderId; + // Check if TOB orders have changed on either side + const tobChanged = + !lastTobOrderId || + lastTobOrderId.bidOrderId !== currentTobOrderIds.bidOrderId || + lastTobOrderId.askOrderId !== currentTobOrderIds.askOrderId; - if (tobChanged) { - // TOB orders changed, update tracking - lastTobOrderIds.set(marketIndex, currentTobOrderIds); - lastTobUpdateTimes.set(marketIndex, currentTime); - logger.debug( - `TOB orders updated for market ${marketIndex}: bidOrderId=${currentTobOrderIds.bidOrderId}, askOrderId=${currentTobOrderIds.askOrderId}` - ); - } else if ( - lastUpdate && - currentTime - lastUpdate > TOB_STUCK_THRESHOLD - ) { - // TOB has been stuck for too long, trigger resubscribe - const stuckDuration = (currentTime - lastUpdate) / 1000; - logger.warn( - `TOB stuck for market ${marketIndex} for ${stuckDuration}s, triggering resubscribe` - ); + if (tobChanged) { + // TOB orders changed, update tracking + lastTobOrderIds.set(marketIndex, currentTobOrderIds); + lastTobUpdateTimes.set(marketIndex, currentTime); + logger.debug( + `TOB orders updated for market ${marketIndex}: bidOrderId=${ + currentTobOrderIds.bidOrderId || 'none' + }, askOrderId=${currentTobOrderIds.askOrderId || 'none'}` + ); + } else if ( + lastUpdate && + currentTime - lastUpdate > TOB_STUCK_THRESHOLD + ) { + // TOB has been stuck for too long, trigger resubscribe + const stuckDuration = (currentTime - lastUpdate) / 1000; + logger.warn( + `TOB stuck for market ${marketIndex} for ${stuckDuration}s, triggering resubscribe` + ); - // Update metrics - tobStuckGauge.setLatestValue(stuckDuration, { - marketIndex: marketIndex.toString(), - marketType: 'perp', - }); + // Update metrics + tobStuckGauge.setLatestValue(stuckDuration, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + }); - // Get the OrderSubscriber instance for resubscribe - if (orderSubscriber) { - try { - // Resubscribe and fetch to clear stuck state - await orderSubscriber.unsubscribe(); - await orderSubscriber.subscribe(); - await orderSubscriber.fetch(); + // Get the OrderSubscriber instance for resubscribe + if (orderSubscriber) { + try { + // Resubscribe and fetch to clear stuck state + await orderSubscriber.unsubscribe(); + await orderSubscriber.subscribe(); + await orderSubscriber.fetch(); - logger.info( - `Successfully resubscribed OrderSubscriber for market ${marketIndex}` - ); - - // Update metrics - tobResubscribeCounter.add(1, { - marketIndex: marketIndex.toString(), - marketType: 'perp', - success: 'true', - }); - - // Reset the timer after successful resubscribe - lastTobUpdateTimes.set(marketIndex, currentTime); - } catch (error) { - logger.error( - `Failed to resubscribe OrderSubscriber for market ${marketIndex}:`, - error - ); - - // Update metrics for failed resubscribe - tobResubscribeCounter.add(1, { - marketIndex: marketIndex.toString(), - marketType: 'perp', - success: 'false', - }); - } - } else { - logger.error( - `OrderSubscriber not available for market ${marketIndex}` + logger.info( + `Successfully resubscribed OrderSubscriber for market ${marketIndex}` ); + + // Update metrics + tobResubscribeCounter.add(1, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + success: 'true', + }); + + // Reset the timer after successful resubscribe + lastTobUpdateTimes.set(marketIndex, currentTime); + } catch (error) { + logger.error( + `Failed to resubscribe OrderSubscriber for market ${marketIndex}:`, + error + ); + + // Update metrics for failed resubscribe + tobResubscribeCounter.add(1, { + marketIndex: marketIndex.toString(), + marketType: 'perp', + success: 'false', + }); } + } else { + logger.error( + `OrderSubscriber not available for market ${marketIndex}` + ); } } } catch (error) {