trade/trade-drift-dlob
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor health checks and prettify/lint

Browse Source
This commit is contained in:
wphan
2025-07-15 10:19:55 -07:00
parent 3cf7afe53e
commit d19e33e840
9 changed files with 171 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

172
src/core/metrics.ts
View File

@@ -39,6 +39,7 @@ enum METRIC_TYPES {
current_system_ts = 'current_system_ts',
health_status = 'health_status',
incoming_requests_count = 'incoming_requests_count',
kinesis_records_sent = 'kinesis_records_sent',
}
export enum HEALTH_STATUS {
@@ -165,72 +166,123 @@ const responseStatusCounter = meter.createCounter(
}
);
const healthCheckInterval = 2000;
let lastHealthCheckSlot = -1;
let lastHealthCheckState = true; // true = healthy, false = unhealthy
let lastHealthCheckPerformed = Date.now() - healthCheckInterval;
let lastTimeHealthy = Date.now() - healthCheckInterval;
const kinesisRecordsSentGauge = meter.createObservableGauge(
METRIC_TYPES.kinesis_records_sent,
{
description: 'Number of records successfully sent to Kinesis stream',
}
);
export const setHealthStatus = (status: HEALTH_STATUS): void => {
healthStatus = status;
let kinesisRecordsSent: KinesisMetric = {
stream: '',
count: 0,
};
const setKinesisRecordsSent = (count: number, stream: string): void => {
kinesisRecordsSent = {
stream,
count,
};
};
kinesisRecordsSentGauge.addCallback((obs: ObservableResult) => {
obs.observe(kinesisRecordsSent.count, { stream: kinesisRecordsSent.stream });
});
/**
* Health check configuration
*/
const HEALTH_CHECK_CONFIG = {
CHECK_INTERVAL_MS: 2000,
// Maximum time allowed between slot updates
MAX_SLOT_STALENESS_MS: 5000,
// Minimum expected slot advancement rate (slots per second)
MIN_SLOT_RATE: 1,
} as const;
/**
* Tracks the health state of the slot subscriber
*/
type HealthState = {
lastSlot: number;
lastSlotTimestamp: number;
};
const globalHealthState: HealthState = {
lastSlot: -1,
lastSlotTimestamp: Date.now(),
};
/**
* Middleware that checks if we are in general healthy by checking that the bulk account loader slot
* has changed recently.
*
* We may be hit by multiple sources performing health checks on us, so this middleware will latch
* to its health state and only update every `healthCheckInterval`.
*
* A grace period is also used to report unhealthy only if we have been unhealthy beyond the grace period.
* Evaluates if the current state is healthy based on slot progression
*/
const handleHealthCheck = (
healthCheckGracePeriod: number,
slotSource: SlotSource
) => {
function evaluateHealth(currentSlot: number): {
isHealthy: boolean;
reason?: string;
} {
const now = Date.now();
// First health check
if (globalHealthState.lastSlot === -1) {
globalHealthState.lastSlot = currentSlot;
globalHealthState.lastSlotTimestamp = now;
return { isHealthy: true };
}
const timeDelta = now - globalHealthState.lastSlotTimestamp;
const slotDelta = currentSlot - globalHealthState.lastSlot;
// Update state if slot has progressed
if (currentSlot > globalHealthState.lastSlot) {
globalHealthState.lastSlot = currentSlot;
globalHealthState.lastSlotTimestamp = now;
}
// Check if slots are too stale
if (timeDelta > HEALTH_CHECK_CONFIG.MAX_SLOT_STALENESS_MS) {
return {
isHealthy: false,
reason: `No slot updates in ${timeDelta}ms (max ${HEALTH_CHECK_CONFIG.MAX_SLOT_STALENESS_MS}ms)`,
};
}
// Check if slot update rate is too low
const slotRate = (slotDelta / timeDelta) * 1000; // Convert to per second
if (slotRate < HEALTH_CHECK_CONFIG.MIN_SLOT_RATE) {
return {
isHealthy: false,
reason: `Slot update rate ${slotRate.toFixed(
2
)} slots/sec below minimum ${HEALTH_CHECK_CONFIG.MIN_SLOT_RATE}`,
};
}
return { isHealthy: true };
}
/**
* Middleware that checks the health of the slot subscriber.
* Health is determined by:
* 1. Regular slot updates
* 2. Minimum slot update rate
*/
const handleHealthCheck = (slotSource: SlotSource) => {
return async (_req, res, _next) => {
if (healthStatus === HEALTH_STATUS.Restart) {
logger.error(`Health status: Restart`);
res.writeHead(500);
res.end(`NOK`);
return;
}
const currentSlot = slotSource.getSlot();
const { isHealthy, reason } = evaluateHealth(currentSlot);
if (Date.now() < lastHealthCheckPerformed + healthCheckInterval) {
if (lastHealthCheckState) {
res.writeHead(200);
res.end('OK');
lastHealthCheckPerformed = Date.now();
return;
}
// always check if last check was unhealthy (give it another chance to recover)
}
// healthy if slot has advanced since the last check
const lastSlotReceived = slotSource.getSlot();
const inGracePeriod =
Date.now() - lastTimeHealthy <= healthCheckGracePeriod;
lastHealthCheckState = lastSlotReceived > lastHealthCheckSlot;
if (!lastHealthCheckState) {
if (!isHealthy) {
logger.error(
`Unhealthy: lastSlot: ${lastSlotReceived}, lastHealthCheckSlot: ${lastHealthCheckSlot}, timeSinceLastCheck: ${
Date.now() - lastHealthCheckPerformed
} ms, sinceLastTimeHealthy: ${
Date.now() - lastTimeHealthy
} ms, inGracePeriod: ${inGracePeriod}`
`Unhealthy: ${reason}, ` +
`Details: currentSlot=${currentSlot}, ` +
`lastSlot=${globalHealthState.lastSlot}, ` +
`timeSinceLastSlot=${
Date.now() - globalHealthState.lastSlotTimestamp
}ms`
);
} else {
lastTimeHealthy = Date.now();
}
lastHealthCheckSlot = lastSlotReceived;
lastHealthCheckPerformed = Date.now();
if (!lastHealthCheckState && !inGracePeriod) {
setHealthStatus(HEALTH_STATUS.UnhealthySlotSubscriber);
res.writeHead(500);
res.end(`NOK`);
res.end('NOK');
return;
}
@@ -240,6 +292,15 @@ const handleHealthCheck = (
};
};
export const setHealthStatus = (status: HEALTH_STATUS): void => {
healthStatus = status;
};
interface KinesisMetric {
stream: string;
count: number;
}
export {
endpointResponseTimeHistogram,
gpaFetchDurationHistogram,
@@ -250,4 +311,5 @@ export {
accountUpdatesCounter,
cacheHitCounter,
runtimeSpecsGauge,
setKinesisRecordsSent,
};

7
src/index.ts
View File

@@ -265,12 +265,9 @@ const main = async (): Promise<void> => {
}
};
app.get(
'/health',
handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, dlobProvider)
);
app.get('/health', handleHealthCheck(dlobProvider));
app.get('/startup', handleStartup);
app.get('/', handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, dlobProvider));
app.get('/', handleHealthCheck(dlobProvider));
app.get('/priorityFees', async (req, res, next) => {
try {

7
src/publishers/dlobPublisher.ts
View File

@@ -652,12 +652,9 @@ const main = async () => {
res.end(await register.metrics());
});
app.get(
'/health',
handleHealthCheck(WS_FALLBACK_FETCH_INTERVAL, dlobProvider)
);
app.get('/health', handleHealthCheck(dlobProvider));
app.get('/startup', handleStartup);
app.get('/', handleHealthCheck(WS_FALLBACK_FETCH_INTERVAL, dlobProvider));
app.get('/', handleHealthCheck(dlobProvider));
const server = app.listen(8080);
// Default keepalive is 5s, since the AWS ALB timeout is 60 seconds, clients

12
src/serverLite.ts
View File

@@ -56,8 +56,6 @@ const sdkConfig = initialize({ env: process.env.ENV });
const stateCommitment: Commitment = 'confirmed';
const serverPort = process.env.PORT || 6969;
export const ORDERBOOK_UPDATE_INTERVAL = 1000;
const WS_FALLBACK_FETCH_INTERVAL = ORDERBOOK_UPDATE_INTERVAL * 60;
const SLOT_STALENESS_TOLERANCE =
parseInt(process.env.SLOT_STALENESS_TOLERANCE) || 100000;
@@ -155,15 +153,9 @@ const main = async (): Promise<void> => {
}
};
app.get(
'/health',
handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, slotSubscriber)
);
app.get('/health', handleHealthCheck(slotSubscriber));
app.get('/startup', handleStartup);
app.get(
'/',
handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, slotSubscriber)
);
app.get('/', handleHealthCheck(slotSubscriber));
app.get('/l3', async (req, res, next) => {
try {

2
src/utils/constants.ts
View File

@@ -1,4 +1,4 @@
import { AuctionParamArgs } from "./types";
import { AuctionParamArgs } from './types';
export const MEASURED_ENDPOINTS = [
'/priorityFees',

5
src/utils/offload.ts
View File

@@ -5,6 +5,7 @@ import {
} from '@aws-sdk/client-kinesis';
import { ConfiguredRetryStrategy } from '@aws-sdk/util-retry';
import { logger } from '../utils/logger';
import { setKinesisRecordsSent } from '../core/metrics';
type EventType = 'DLOBSnapshot' | 'DLOBL3Snapshot';
@@ -83,9 +84,7 @@ export const OffloadQueue = () => {
}
const successCount = batch.length - (response.FailedRecordCount || 0);
logger.info(
`Successfully sent ${successCount} records to Kinesis stream: ${kinesisStream}`
);
setKinesisRecordsSent(successCount, kinesisStream);
} catch (error) {
logger.error('Error processing queue:', error);
} finally {

35
src/utils/tests/auctionParams.test.ts
View File

@@ -5,7 +5,7 @@ import {
MarketType,
ZERO,
QUOTE_PRECISION,
PRICE_PRECISION
PRICE_PRECISION,
} from '@drift-labs/sdk';
import {
createMarketBasedAuctionParams,
@@ -16,7 +16,7 @@ import {
describe('Auction Parameters Functions', () => {
describe('createMarketBasedAuctionParams', () => {
it('should apply market-based logic for major PERP markets (0, 1, 2)', () => {
[0, 1, 2].forEach(marketIndex => {
[0, 1, 2].forEach((marketIndex) => {
const args = {
marketIndex,
marketType: 'perp' as any,
@@ -35,7 +35,7 @@ describe('Auction Parameters Functions', () => {
});
it('should apply market-based logic for minor PERP markets (>2)', () => {
[3, 4, 5, 10].forEach(marketIndex => {
[3, 4, 5, 10].forEach((marketIndex) => {
const args = {
marketIndex,
marketType: 'perp' as any,
@@ -54,7 +54,7 @@ describe('Auction Parameters Functions', () => {
});
it('should apply market-based logic for SPOT markets when "marketBased" is passed', () => {
[0, 1, 2, 5, 10].forEach(marketIndex => {
[0, 1, 2, 5, 10].forEach((marketIndex) => {
const args = {
marketIndex,
marketType: 'spot' as any,
@@ -381,7 +381,9 @@ describe('Auction Parameters Functions', () => {
);
expect(result.success).toBe(true);
expect(result.data?.marketOrderParams.direction).toBe(PositionDirection.SHORT);
expect(result.data?.marketOrderParams.direction).toBe(
PositionDirection.SHORT
);
expect(result.data?.estimatedPrices.entryPrice.gt(ZERO)).toBe(true);
});
@@ -397,12 +399,12 @@ describe('Auction Parameters Functions', () => {
if (key.includes('dlob_orderbook')) {
return {
bids: [
{ price: '159950000', size: '500000000' }, // $159.95, 0.5 SOL
{ price: '159950000', size: '500000000' }, // $159.95, 0.5 SOL
{ price: '159900000', size: '1000000000' }, // $159.90, 1 SOL
{ price: '159850000', size: '2000000000' }, // $159.85, 2 SOL
],
asks: [
{ price: '160050000', size: '500000000' }, // $160.05, 0.5 SOL
{ price: '160050000', size: '500000000' }, // $160.05, 0.5 SOL
{ price: '160100000', size: '1000000000' }, // $160.10, 1 SOL
{ price: '160150000', size: '2000000000' }, // $160.15, 2 SOL
],
@@ -444,7 +446,11 @@ describe('Auction Parameters Functions', () => {
expect(largeResult.data?.estimatedPrices.entryPrice.gt(ZERO)).toBe(true);
// Large orders should have higher price impact
expect(largeResult.data?.estimatedPrices.priceImpact.gte(smallResult.data?.estimatedPrices.priceImpact)).toBe(true);
expect(
largeResult.data?.estimatedPrices.priceImpact.gte(
smallResult.data?.estimatedPrices.priceImpact
)
).toBe(true);
});
it('should handle zero oracle price scenario', async () => {
@@ -600,11 +606,15 @@ describe('Auction Parameters Functions', () => {
},
];
scenarios.forEach(scenario => {
scenarios.forEach((scenario) => {
const result = createMarketBasedAuctionParams(scenario.input);
expect(result.auctionStartPriceOffsetFrom).toBe(scenario.expectedOffset.from);
expect(result.auctionStartPriceOffset).toBe(scenario.expectedOffset.value);
expect(result.auctionStartPriceOffsetFrom).toBe(
scenario.expectedOffset.from
);
expect(result.auctionStartPriceOffset).toBe(
scenario.expectedOffset.value
);
// Verify all original parameters are preserved
expect(result.marketIndex).toBe(scenario.input.marketIndex);
@@ -666,7 +676,8 @@ describe('Auction Parameters Functions', () => {
isOracleOrder: true,
};
const formattedResponse = formatAuctionParamsForResponse(mockDerivedParams);
const formattedResponse =
formatAuctionParamsForResponse(mockDerivedParams);
// Verify the complete flow
expect(formattedResponse.auctionStartPriceOffsetFrom).toBe('mark'); // Market-based for major market

4
src/utils/types.ts
View File

@@ -1,5 +1,5 @@
import { AssetType, MarketTypeStr } from "@drift-labs/sdk";
import { TradeOffsetPrice } from "@drift/common";
import { AssetType, MarketTypeStr } from '@drift-labs/sdk';
import { TradeOffsetPrice } from '@drift/common';
export type AuctionParamArgs = {
// mandatory args

6
src/utils/utils.ts
View File

@@ -656,14 +656,16 @@ export function createMarketBasedAuctionParams(
// Resolve "marketBased" values and undefined values (both should use market-based logic)
const resolvedAuctionStartPriceOffsetFrom =
args.auctionStartPriceOffsetFrom === 'marketBased' || args.auctionStartPriceOffsetFrom === undefined
args.auctionStartPriceOffsetFrom === 'marketBased' ||
args.auctionStartPriceOffsetFrom === undefined
? isMajorMarket
? 'mark'
: 'bestOffer'
: args.auctionStartPriceOffsetFrom;
const resolvedAuctionStartPriceOffset =
args.auctionStartPriceOffset === 'marketBased' || args.auctionStartPriceOffset === undefined
args.auctionStartPriceOffset === 'marketBased' ||
args.auctionStartPriceOffset === undefined
? isMajorMarket
? 0
: -0.1