more health check fixes, check slot rate

This commit is contained in:
wphan
2025-07-15 13:06:42 -07:00
parent f0d906baca
commit 508aef4f2e
3 changed files with 23 additions and 20 deletions

View File

@@ -13,7 +13,7 @@ const HEALTH_CHECK_CONFIG = {
CHECK_INTERVAL_MS: 2000,
// Maximum time allowed between slot updates
MAX_SLOT_STALENESS_MS: 5000,
// Minimum expected slot advancement rate (slots per second)
// Minimum expected slot advancement rate (slots slower than 1 per second is problematic)
MIN_SLOT_RATE: 1,
} as const;
@@ -49,13 +49,27 @@ function evaluateHealth(currentSlot: number): {
const timeDelta = now - globalHealthState.lastSlotTimestamp;
const slotDelta = currentSlot - globalHealthState.lastSlot;
// Update state if slot has progressed
// If slot has progressed, we are healthy, check rate and update state.
if (currentSlot > globalHealthState.lastSlot) {
// Update state
globalHealthState.lastSlot = currentSlot;
globalHealthState.lastSlotTimestamp = now;
// Check if slot update rate is too low
const slotRate = (slotDelta / timeDelta) * 1000; // Convert to per second
if (slotRate < HEALTH_CHECK_CONFIG.MIN_SLOT_RATE) {
return {
isHealthy: false,
reason: `Slot update rate ${slotRate.toFixed(
2
)} slots/sec below minimum ${HEALTH_CHECK_CONFIG.MIN_SLOT_RATE}`,
};
}
return { isHealthy: true };
}
// Check if slots are too stale
// If slot has NOT progressed, check for staleness.
if (timeDelta > HEALTH_CHECK_CONFIG.MAX_SLOT_STALENESS_MS) {
return {
isHealthy: false,
@@ -63,17 +77,7 @@ function evaluateHealth(currentSlot: number): {
};
}
// Check if slot update rate is too low
const slotRate = (slotDelta / timeDelta) * 1000; // Convert to per second
if (slotRate < HEALTH_CHECK_CONFIG.MIN_SLOT_RATE) {
return {
isHealthy: false,
reason: `Slot update rate ${slotRate.toFixed(
2
)} slots/sec below minimum ${HEALTH_CHECK_CONFIG.MIN_SLOT_RATE}`,
};
}
// Slot has not progressed, but not stale yet. Still healthy.
return { isHealthy: true };
}

View File

@@ -1,12 +1,12 @@
import { describe, it, expect, jest, beforeEach } from '@jest/globals';
import {
BN,
PositionDirection,
MarketType,
ZERO,
QUOTE_PRECISION,
PositionDirection,
MarketType,
ZERO,
QUOTE_PRECISION,
PRICE_PRECISION,
BASE_PRECISION
BASE_PRECISION,
} from '@drift-labs/sdk';
import {
createMarketBasedAuctionParams,

View File

@@ -22,7 +22,6 @@ import {
AssetType,
MainnetSpotMarkets,
DevnetSpotMarkets,
QUOTE_PRECISION,
PERCENTAGE_PRECISION_EXP,
} from '@drift-labs/sdk';
import { RedisClient } from '@drift/common/clients';