trade/trade-drift-dlob
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor health checks and prettify/lint

Browse Source
This commit is contained in:
wphan
2025-07-15 10:19:55 -07:00
parent 3cf7afe53e
commit d19e33e840
9 changed files with 171 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

172
src/core/metrics.ts
View File

@@ -39,6 +39,7 @@ enum METRIC_TYPES {
current_system_ts = 'current_system_ts', current_system_ts = 'current_system_ts',
health_status = 'health_status', health_status = 'health_status',
incoming_requests_count = 'incoming_requests_count', incoming_requests_count = 'incoming_requests_count',
kinesis_records_sent = 'kinesis_records_sent',
} }
export enum HEALTH_STATUS { export enum HEALTH_STATUS {
@@ -165,72 +166,123 @@ const responseStatusCounter = meter.createCounter(
} }
); );
const healthCheckInterval = 2000; const kinesisRecordsSentGauge = meter.createObservableGauge(
let lastHealthCheckSlot = -1; METRIC_TYPES.kinesis_records_sent,
let lastHealthCheckState = true; // true = healthy, false = unhealthy {
let lastHealthCheckPerformed = Date.now() - healthCheckInterval; description: 'Number of records successfully sent to Kinesis stream',
let lastTimeHealthy = Date.now() - healthCheckInterval; }
);
export const setHealthStatus = (status: HEALTH_STATUS): void => { let kinesisRecordsSent: KinesisMetric = {
healthStatus = status; stream: '',
count: 0,
};
const setKinesisRecordsSent = (count: number, stream: string): void => {
kinesisRecordsSent = {
stream,
count,
};
};
kinesisRecordsSentGauge.addCallback((obs: ObservableResult) => {
obs.observe(kinesisRecordsSent.count, { stream: kinesisRecordsSent.stream });
});
/**
* Health check configuration
*/
const HEALTH_CHECK_CONFIG = {
CHECK_INTERVAL_MS: 2000,
// Maximum time allowed between slot updates
MAX_SLOT_STALENESS_MS: 5000,
// Minimum expected slot advancement rate (slots per second)
MIN_SLOT_RATE: 1,
} as const;
/**
* Tracks the health state of the slot subscriber
*/
type HealthState = {
lastSlot: number;
lastSlotTimestamp: number;
};
const globalHealthState: HealthState = {
lastSlot: -1,
lastSlotTimestamp: Date.now(),
}; };
/** /**
* Middleware that checks if we are in general healthy by checking that the bulk account loader slot * Evaluates if the current state is healthy based on slot progression
* has changed recently.
*
* We may be hit by multiple sources performing health checks on us, so this middleware will latch
* to its health state and only update every `healthCheckInterval`.
*
* A grace period is also used to report unhealthy only if we have been unhealthy beyond the grace period.
*/ */
const handleHealthCheck = ( function evaluateHealth(currentSlot: number): {
healthCheckGracePeriod: number, isHealthy: boolean;
slotSource: SlotSource reason?: string;
) => { } {
const now = Date.now();
// First health check
if (globalHealthState.lastSlot === -1) {
globalHealthState.lastSlot = currentSlot;
globalHealthState.lastSlotTimestamp = now;
return { isHealthy: true };
}
const timeDelta = now - globalHealthState.lastSlotTimestamp;
const slotDelta = currentSlot - globalHealthState.lastSlot;
// Update state if slot has progressed
if (currentSlot > globalHealthState.lastSlot) {
globalHealthState.lastSlot = currentSlot;
globalHealthState.lastSlotTimestamp = now;
}
// Check if slots are too stale
if (timeDelta > HEALTH_CHECK_CONFIG.MAX_SLOT_STALENESS_MS) {
return {
isHealthy: false,
reason: `No slot updates in ${timeDelta}ms (max ${HEALTH_CHECK_CONFIG.MAX_SLOT_STALENESS_MS}ms)`,
};
}
// Check if slot update rate is too low
const slotRate = (slotDelta / timeDelta) * 1000; // Convert to per second
if (slotRate < HEALTH_CHECK_CONFIG.MIN_SLOT_RATE) {
return {
isHealthy: false,
reason: `Slot update rate ${slotRate.toFixed(
2
)} slots/sec below minimum ${HEALTH_CHECK_CONFIG.MIN_SLOT_RATE}`,
};
}
return { isHealthy: true };
}
/**
* Middleware that checks the health of the slot subscriber.
* Health is determined by:
* 1. Regular slot updates
* 2. Minimum slot update rate
*/
const handleHealthCheck = (slotSource: SlotSource) => {
return async (_req, res, _next) => { return async (_req, res, _next) => {
if (healthStatus === HEALTH_STATUS.Restart) { const currentSlot = slotSource.getSlot();
logger.error(`Health status: Restart`); const { isHealthy, reason } = evaluateHealth(currentSlot);
res.writeHead(500);
res.end(`NOK`);
return;
}
if (Date.now() < lastHealthCheckPerformed + healthCheckInterval) { if (!isHealthy) {
if (lastHealthCheckState) {
res.writeHead(200);
res.end('OK');
lastHealthCheckPerformed = Date.now();
return;
}
// always check if last check was unhealthy (give it another chance to recover)
}
// healthy if slot has advanced since the last check
const lastSlotReceived = slotSource.getSlot();
const inGracePeriod =
Date.now() - lastTimeHealthy <= healthCheckGracePeriod;
lastHealthCheckState = lastSlotReceived > lastHealthCheckSlot;
if (!lastHealthCheckState) {
logger.error( logger.error(
`Unhealthy: lastSlot: ${lastSlotReceived}, lastHealthCheckSlot: ${lastHealthCheckSlot}, timeSinceLastCheck: ${ `Unhealthy: ${reason}, ` +
Date.now() - lastHealthCheckPerformed `Details: currentSlot=${currentSlot}, ` +
} ms, sinceLastTimeHealthy: ${ `lastSlot=${globalHealthState.lastSlot}, ` +
Date.now() - lastTimeHealthy `timeSinceLastSlot=${
} ms, inGracePeriod: ${inGracePeriod}` Date.now() - globalHealthState.lastSlotTimestamp
}ms`
); );
} else {
lastTimeHealthy = Date.now();
}
lastHealthCheckSlot = lastSlotReceived;
lastHealthCheckPerformed = Date.now();
if (!lastHealthCheckState && !inGracePeriod) {
setHealthStatus(HEALTH_STATUS.UnhealthySlotSubscriber); setHealthStatus(HEALTH_STATUS.UnhealthySlotSubscriber);
res.writeHead(500); res.writeHead(500);
res.end(`NOK`); res.end('NOK');
return; return;
} }
@@ -240,6 +292,15 @@ const handleHealthCheck = (
}; };
}; };
export const setHealthStatus = (status: HEALTH_STATUS): void => {
healthStatus = status;
};
interface KinesisMetric {
stream: string;
count: number;
}
export { export {
endpointResponseTimeHistogram, endpointResponseTimeHistogram,
gpaFetchDurationHistogram, gpaFetchDurationHistogram,
@@ -250,4 +311,5 @@ export {
accountUpdatesCounter, accountUpdatesCounter,
cacheHitCounter, cacheHitCounter,
runtimeSpecsGauge, runtimeSpecsGauge,
setKinesisRecordsSent,
}; };

9
src/index.ts
View File

@@ -265,12 +265,9 @@ const main = async (): Promise<void> => {
} }
}; };
app.get( app.get('/health', handleHealthCheck(dlobProvider));
'/health',
handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, dlobProvider)
);
app.get('/startup', handleStartup); app.get('/startup', handleStartup);
app.get('/', handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, dlobProvider)); app.get('/', handleHealthCheck(dlobProvider));
app.get('/priorityFees', async (req, res, next) => { app.get('/priorityFees', async (req, res, next) => {
try { try {
@@ -1065,4 +1062,4 @@ async function recursiveTryCatch(f: () => Promise<void>) {
recursiveTryCatch(() => main()); recursiveTryCatch(() => main());
export { commitHash, driftClient, driftEnv, endpoint, sdkConfig, wsEndpoint }; export { commitHash, driftClient, driftEnv, endpoint, sdkConfig, wsEndpoint };

7
src/publishers/dlobPublisher.ts
View File

@@ -652,12 +652,9 @@ const main = async () => {
res.end(await register.metrics()); res.end(await register.metrics());
}); });
app.get( app.get('/health', handleHealthCheck(dlobProvider));
'/health',
handleHealthCheck(WS_FALLBACK_FETCH_INTERVAL, dlobProvider)
);
app.get('/startup', handleStartup); app.get('/startup', handleStartup);
app.get('/', handleHealthCheck(WS_FALLBACK_FETCH_INTERVAL, dlobProvider)); app.get('/', handleHealthCheck(dlobProvider));
const server = app.listen(8080); const server = app.listen(8080);
// Default keepalive is 5s, since the AWS ALB timeout is 60 seconds, clients // Default keepalive is 5s, since the AWS ALB timeout is 60 seconds, clients

12
src/serverLite.ts
View File

@@ -56,8 +56,6 @@ const sdkConfig = initialize({ env: process.env.ENV });
const stateCommitment: Commitment = 'confirmed'; const stateCommitment: Commitment = 'confirmed';
const serverPort = process.env.PORT || 6969; const serverPort = process.env.PORT || 6969;
export const ORDERBOOK_UPDATE_INTERVAL = 1000;
const WS_FALLBACK_FETCH_INTERVAL = ORDERBOOK_UPDATE_INTERVAL * 60;
const SLOT_STALENESS_TOLERANCE = const SLOT_STALENESS_TOLERANCE =
parseInt(process.env.SLOT_STALENESS_TOLERANCE) || 100000; parseInt(process.env.SLOT_STALENESS_TOLERANCE) || 100000;
@@ -155,15 +153,9 @@ const main = async (): Promise<void> => {
} }
}; };
app.get( app.get('/health', handleHealthCheck(slotSubscriber));
'/health',
handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, slotSubscriber)
);
app.get('/startup', handleStartup); app.get('/startup', handleStartup);
app.get( app.get('/', handleHealthCheck(slotSubscriber));
'/',
handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, slotSubscriber)
);
app.get('/l3', async (req, res, next) => { app.get('/l3', async (req, res, next) => {
try { try {

2
src/utils/constants.ts
View File

@@ -1,4 +1,4 @@
import { AuctionParamArgs } from "./types"; import { AuctionParamArgs } from './types';
export const MEASURED_ENDPOINTS = [ export const MEASURED_ENDPOINTS = [
'/priorityFees', '/priorityFees',

5
src/utils/offload.ts
View File

@@ -5,6 +5,7 @@ import {
} from '@aws-sdk/client-kinesis'; } from '@aws-sdk/client-kinesis';
import { ConfiguredRetryStrategy } from '@aws-sdk/util-retry'; import { ConfiguredRetryStrategy } from '@aws-sdk/util-retry';
import { logger } from '../utils/logger'; import { logger } from '../utils/logger';
import { setKinesisRecordsSent } from '../core/metrics';
type EventType = 'DLOBSnapshot' | 'DLOBL3Snapshot'; type EventType = 'DLOBSnapshot' | 'DLOBL3Snapshot';
@@ -83,9 +84,7 @@ export const OffloadQueue = () => {
} }
const successCount = batch.length - (response.FailedRecordCount || 0); const successCount = batch.length - (response.FailedRecordCount || 0);
logger.info( setKinesisRecordsSent(successCount, kinesisStream);
`Successfully sent ${successCount} records to Kinesis stream: ${kinesisStream}`
);
} catch (error) { } catch (error) {
logger.error('Error processing queue:', error); logger.error('Error processing queue:', error);
} finally { } finally {

63
src/utils/tests/auctionParams.test.ts
View File

@@ -1,11 +1,11 @@
import { describe, it, expect, jest, beforeEach } from '@jest/globals'; import { describe, it, expect, jest, beforeEach } from '@jest/globals';
import { import {
BN, BN,
PositionDirection, PositionDirection,
MarketType, MarketType,
ZERO, ZERO,
QUOTE_PRECISION, QUOTE_PRECISION,
PRICE_PRECISION PRICE_PRECISION,
} from '@drift-labs/sdk'; } from '@drift-labs/sdk';
import { import {
createMarketBasedAuctionParams, createMarketBasedAuctionParams,
@@ -16,7 +16,7 @@ import {
describe('Auction Parameters Functions', () => { describe('Auction Parameters Functions', () => {
describe('createMarketBasedAuctionParams', () => { describe('createMarketBasedAuctionParams', () => {
it('should apply market-based logic for major PERP markets (0, 1, 2)', () => { it('should apply market-based logic for major PERP markets (0, 1, 2)', () => {
[0, 1, 2].forEach(marketIndex => { [0, 1, 2].forEach((marketIndex) => {
const args = { const args = {
marketIndex, marketIndex,
marketType: 'perp' as any, marketType: 'perp' as any,
@@ -35,7 +35,7 @@ describe('Auction Parameters Functions', () => {
}); });
it('should apply market-based logic for minor PERP markets (>2)', () => { it('should apply market-based logic for minor PERP markets (>2)', () => {
[3, 4, 5, 10].forEach(marketIndex => { [3, 4, 5, 10].forEach((marketIndex) => {
const args = { const args = {
marketIndex, marketIndex,
marketType: 'perp' as any, marketType: 'perp' as any,
@@ -54,7 +54,7 @@ describe('Auction Parameters Functions', () => {
}); });
it('should apply market-based logic for SPOT markets when "marketBased" is passed', () => { it('should apply market-based logic for SPOT markets when "marketBased" is passed', () => {
[0, 1, 2, 5, 10].forEach(marketIndex => { [0, 1, 2, 5, 10].forEach((marketIndex) => {
const args = { const args = {
marketIndex, marketIndex,
marketType: 'spot' as any, marketType: 'spot' as any,
@@ -206,7 +206,7 @@ describe('Auction Parameters Functions', () => {
it('should successfully calculate prices with mock L2 orderbook data', async () => { it('should successfully calculate prices with mock L2 orderbook data', async () => {
const solPrice = 160; // $160 SOL price const solPrice = 160; // $160 SOL price
mockDriftClient.getOracleDataForPerpMarket.mockReturnValue({ mockDriftClient.getOracleDataForPerpMarket.mockReturnValue({
price: new BN(solPrice).mul(PRICE_PRECISION), price: new BN(solPrice).mul(PRICE_PRECISION),
}); });
@@ -294,7 +294,7 @@ describe('Auction Parameters Functions', () => {
); );
expect(result.success).toBe(true); expect(result.success).toBe(true);
// With $1,000 at $160 per SOL, we should get ~6.25 SOL // With $1,000 at $160 per SOL, we should get ~6.25 SOL
const baseAmount = result.data?.marketOrderParams.baseAmount; const baseAmount = result.data?.marketOrderParams.baseAmount;
expect(baseAmount).toBeDefined(); expect(baseAmount).toBeDefined();
@@ -311,7 +311,7 @@ describe('Auction Parameters Functions', () => {
it('should handle different market types correctly', async () => { it('should handle different market types correctly', async () => {
const usdcPrice = 1; // $1 USDC price const usdcPrice = 1; // $1 USDC price
mockDriftClient.getOracleDataForSpotMarket.mockReturnValue({ mockDriftClient.getOracleDataForSpotMarket.mockReturnValue({
price: new BN(usdcPrice).mul(PRICE_PRECISION), price: new BN(usdcPrice).mul(PRICE_PRECISION),
}); });
@@ -349,7 +349,7 @@ describe('Auction Parameters Functions', () => {
it('should handle different directions correctly', async () => { it('should handle different directions correctly', async () => {
const solPrice = 160; // $160 SOL price const solPrice = 160; // $160 SOL price
mockDriftClient.getOracleDataForPerpMarket.mockReturnValue({ mockDriftClient.getOracleDataForPerpMarket.mockReturnValue({
price: new BN(solPrice).mul(PRICE_PRECISION), price: new BN(solPrice).mul(PRICE_PRECISION),
}); });
@@ -381,13 +381,15 @@ describe('Auction Parameters Functions', () => {
); );
expect(result.success).toBe(true); expect(result.success).toBe(true);
expect(result.data?.marketOrderParams.direction).toBe(PositionDirection.SHORT); expect(result.data?.marketOrderParams.direction).toBe(
PositionDirection.SHORT
);
expect(result.data?.estimatedPrices.entryPrice.gt(ZERO)).toBe(true); expect(result.data?.estimatedPrices.entryPrice.gt(ZERO)).toBe(true);
}); });
it('should handle various order sizes with L2 depth', async () => { it('should handle various order sizes with L2 depth', async () => {
const solPrice = 160; // $160 SOL price const solPrice = 160; // $160 SOL price
mockDriftClient.getOracleDataForPerpMarket.mockReturnValue({ mockDriftClient.getOracleDataForPerpMarket.mockReturnValue({
price: new BN(solPrice).mul(PRICE_PRECISION), price: new BN(solPrice).mul(PRICE_PRECISION),
}); });
@@ -397,12 +399,12 @@ describe('Auction Parameters Functions', () => {
if (key.includes('dlob_orderbook')) { if (key.includes('dlob_orderbook')) {
return { return {
bids: [ bids: [
{ price: '159950000', size: '500000000' }, // $159.95, 0.5 SOL { price: '159950000', size: '500000000' }, // $159.95, 0.5 SOL
{ price: '159900000', size: '1000000000' }, // $159.90, 1 SOL { price: '159900000', size: '1000000000' }, // $159.90, 1 SOL
{ price: '159850000', size: '2000000000' }, // $159.85, 2 SOL { price: '159850000', size: '2000000000' }, // $159.85, 2 SOL
], ],
asks: [ asks: [
{ price: '160050000', size: '500000000' }, // $160.05, 0.5 SOL { price: '160050000', size: '500000000' }, // $160.05, 0.5 SOL
{ price: '160100000', size: '1000000000' }, // $160.10, 1 SOL { price: '160100000', size: '1000000000' }, // $160.10, 1 SOL
{ price: '160150000', size: '2000000000' }, // $160.15, 2 SOL { price: '160150000', size: '2000000000' }, // $160.15, 2 SOL
], ],
@@ -442,9 +444,13 @@ describe('Auction Parameters Functions', () => {
expect(largeResult.success).toBe(true); expect(largeResult.success).toBe(true);
expect(largeResult.data?.estimatedPrices.entryPrice.gt(ZERO)).toBe(true); expect(largeResult.data?.estimatedPrices.entryPrice.gt(ZERO)).toBe(true);
// Large orders should have higher price impact // Large orders should have higher price impact
expect(largeResult.data?.estimatedPrices.priceImpact.gte(smallResult.data?.estimatedPrices.priceImpact)).toBe(true); expect(
largeResult.data?.estimatedPrices.priceImpact.gte(
smallResult.data?.estimatedPrices.priceImpact
)
).toBe(true);
}); });
it('should handle zero oracle price scenario', async () => { it('should handle zero oracle price scenario', async () => {
@@ -600,12 +606,16 @@ describe('Auction Parameters Functions', () => {
}, },
]; ];
scenarios.forEach(scenario => { scenarios.forEach((scenario) => {
const result = createMarketBasedAuctionParams(scenario.input); const result = createMarketBasedAuctionParams(scenario.input);
expect(result.auctionStartPriceOffsetFrom).toBe(scenario.expectedOffset.from); expect(result.auctionStartPriceOffsetFrom).toBe(
expect(result.auctionStartPriceOffset).toBe(scenario.expectedOffset.value); scenario.expectedOffset.from
);
expect(result.auctionStartPriceOffset).toBe(
scenario.expectedOffset.value
);
// Verify all original parameters are preserved // Verify all original parameters are preserved
expect(result.marketIndex).toBe(scenario.input.marketIndex); expect(result.marketIndex).toBe(scenario.input.marketIndex);
expect(result.marketType).toBe(scenario.input.marketType); expect(result.marketType).toBe(scenario.input.marketType);
@@ -652,7 +662,7 @@ describe('Auction Parameters Functions', () => {
}; };
const auctionParams = createMarketBasedAuctionParams(input); const auctionParams = createMarketBasedAuctionParams(input);
// Simulate the structure that would come from deriveMarketOrderParams // Simulate the structure that would come from deriveMarketOrderParams
const mockDerivedParams = { const mockDerivedParams = {
auctionStartPriceOffsetFrom: auctionParams.auctionStartPriceOffsetFrom, auctionStartPriceOffsetFrom: auctionParams.auctionStartPriceOffsetFrom,
@@ -666,7 +676,8 @@ describe('Auction Parameters Functions', () => {
isOracleOrder: true, isOracleOrder: true,
}; };
const formattedResponse = formatAuctionParamsForResponse(mockDerivedParams); const formattedResponse =
formatAuctionParamsForResponse(mockDerivedParams);
// Verify the complete flow // Verify the complete flow
expect(formattedResponse.auctionStartPriceOffsetFrom).toBe('mark'); // Market-based for major market expect(formattedResponse.auctionStartPriceOffsetFrom).toBe('mark'); // Market-based for major market

6
src/utils/types.ts
View File

@@ -1,5 +1,5 @@
import { AssetType, MarketTypeStr } from "@drift-labs/sdk"; import { AssetType, MarketTypeStr } from '@drift-labs/sdk';
import { TradeOffsetPrice } from "@drift/common"; import { TradeOffsetPrice } from '@drift/common';
export type AuctionParamArgs = { export type AuctionParamArgs = {
// mandatory args // mandatory args
@@ -21,4 +21,4 @@ export type AuctionParamArgs = {
auctionEndPriceOffsetFrom?: TradeOffsetPrice; auctionEndPriceOffsetFrom?: TradeOffsetPrice;
additionalEndPriceBuffer?: string; additionalEndPriceBuffer?: string;
userOrderId?: number; userOrderId?: number;
}; };

6
src/utils/utils.ts
View File

@@ -656,14 +656,16 @@ export function createMarketBasedAuctionParams(
// Resolve "marketBased" values and undefined values (both should use market-based logic) // Resolve "marketBased" values and undefined values (both should use market-based logic)
const resolvedAuctionStartPriceOffsetFrom = const resolvedAuctionStartPriceOffsetFrom =
args.auctionStartPriceOffsetFrom === 'marketBased' || args.auctionStartPriceOffsetFrom === undefined args.auctionStartPriceOffsetFrom === 'marketBased' ||
args.auctionStartPriceOffsetFrom === undefined
? isMajorMarket ? isMajorMarket
? 'mark' ? 'mark'
: 'bestOffer' : 'bestOffer'
: args.auctionStartPriceOffsetFrom; : args.auctionStartPriceOffsetFrom;
const resolvedAuctionStartPriceOffset = const resolvedAuctionStartPriceOffset =
args.auctionStartPriceOffset === 'marketBased' || args.auctionStartPriceOffset === undefined args.auctionStartPriceOffset === 'marketBased' ||
args.auctionStartPriceOffset === undefined
? isMajorMarket ? isMajorMarket
? 0 ? 0
: -0.1 : -0.1