more health check fixes, check slot rate

This commit is contained in:
wphan
2025-07-15 13:06:42 -07:00
parent f0d906baca
commit 508aef4f2e
3 changed files with 23 additions and 20 deletions

View File

@@ -13,7 +13,7 @@ const HEALTH_CHECK_CONFIG = {
CHECK_INTERVAL_MS: 2000,
// Maximum time allowed between slot updates
MAX_SLOT_STALENESS_MS: 5000,
// Minimum expected slot advancement rate (slots per second)
// Minimum expected slot advancement rate (slots slower than 1 per second is problematic)
MIN_SLOT_RATE: 1,
} as const;
@@ -49,19 +49,11 @@ function evaluateHealth(currentSlot: number): {
const timeDelta = now - globalHealthState.lastSlotTimestamp;
const slotDelta = currentSlot - globalHealthState.lastSlot;
// Update state if slot has progressed
// If slot has progressed, we are healthy, check rate and update state.
if (currentSlot > globalHealthState.lastSlot) {
// Update state
globalHealthState.lastSlot = currentSlot;
globalHealthState.lastSlotTimestamp = now;
}
// Check if slots are too stale
if (timeDelta > HEALTH_CHECK_CONFIG.MAX_SLOT_STALENESS_MS) {
return {
isHealthy: false,
reason: `No slot updates in ${timeDelta}ms (max ${HEALTH_CHECK_CONFIG.MAX_SLOT_STALENESS_MS}ms)`,
};
}
// Check if slot update rate is too low
const slotRate = (slotDelta / timeDelta) * 1000; // Convert to per second
@@ -77,6 +69,18 @@ function evaluateHealth(currentSlot: number): {
return { isHealthy: true };
}
// If slot has NOT progressed, check for staleness.
if (timeDelta > HEALTH_CHECK_CONFIG.MAX_SLOT_STALENESS_MS) {
return {
isHealthy: false,
reason: `No slot updates in ${timeDelta}ms (max ${HEALTH_CHECK_CONFIG.MAX_SLOT_STALENESS_MS}ms)`,
};
}
// Slot has not progressed, but not stale yet. Still healthy.
return { isHealthy: true };
}
let healthStatus: HEALTH_STATUS = HEALTH_STATUS.Ok;
const setHealthStatus = (status: HEALTH_STATUS): void => {
healthStatus = status;

View File

@@ -6,7 +6,7 @@ import {
ZERO,
QUOTE_PRECISION,
PRICE_PRECISION,
BASE_PRECISION
BASE_PRECISION,
} from '@drift-labs/sdk';
import {
createMarketBasedAuctionParams,

View File

@@ -22,7 +22,6 @@ import {
AssetType,
MainnetSpotMarkets,
DevnetSpotMarkets,
QUOTE_PRECISION,
PERCENTAGE_PRECISION_EXP,
} from '@drift-labs/sdk';
import { RedisClient } from '@drift/common/clients';