more health check fixes, check slot rate

This commit is contained in:
wphan
2025-07-15 13:06:42 -07:00
parent f0d906baca
commit 508aef4f2e
3 changed files with 23 additions and 20 deletions

View File

@@ -13,7 +13,7 @@ const HEALTH_CHECK_CONFIG = {
CHECK_INTERVAL_MS: 2000, CHECK_INTERVAL_MS: 2000,
// Maximum time allowed between slot updates // Maximum time allowed between slot updates
MAX_SLOT_STALENESS_MS: 5000, MAX_SLOT_STALENESS_MS: 5000,
// Minimum expected slot advancement rate (slots per second) // Minimum expected slot advancement rate (slots slower than 1 per second is problematic)
MIN_SLOT_RATE: 1, MIN_SLOT_RATE: 1,
} as const; } as const;
@@ -49,19 +49,11 @@ function evaluateHealth(currentSlot: number): {
const timeDelta = now - globalHealthState.lastSlotTimestamp; const timeDelta = now - globalHealthState.lastSlotTimestamp;
const slotDelta = currentSlot - globalHealthState.lastSlot; const slotDelta = currentSlot - globalHealthState.lastSlot;
// Update state if slot has progressed // If slot has progressed, we are healthy, check rate and update state.
if (currentSlot > globalHealthState.lastSlot) { if (currentSlot > globalHealthState.lastSlot) {
// Update state
globalHealthState.lastSlot = currentSlot; globalHealthState.lastSlot = currentSlot;
globalHealthState.lastSlotTimestamp = now; globalHealthState.lastSlotTimestamp = now;
}
// Check if slots are too stale
if (timeDelta > HEALTH_CHECK_CONFIG.MAX_SLOT_STALENESS_MS) {
return {
isHealthy: false,
reason: `No slot updates in ${timeDelta}ms (max ${HEALTH_CHECK_CONFIG.MAX_SLOT_STALENESS_MS}ms)`,
};
}
// Check if slot update rate is too low // Check if slot update rate is too low
const slotRate = (slotDelta / timeDelta) * 1000; // Convert to per second const slotRate = (slotDelta / timeDelta) * 1000; // Convert to per second
@@ -77,6 +69,18 @@ function evaluateHealth(currentSlot: number): {
return { isHealthy: true }; return { isHealthy: true };
} }
// If slot has NOT progressed, check for staleness.
if (timeDelta > HEALTH_CHECK_CONFIG.MAX_SLOT_STALENESS_MS) {
return {
isHealthy: false,
reason: `No slot updates in ${timeDelta}ms (max ${HEALTH_CHECK_CONFIG.MAX_SLOT_STALENESS_MS}ms)`,
};
}
// Slot has not progressed, but not stale yet. Still healthy.
return { isHealthy: true };
}
let healthStatus: HEALTH_STATUS = HEALTH_STATUS.Ok; let healthStatus: HEALTH_STATUS = HEALTH_STATUS.Ok;
const setHealthStatus = (status: HEALTH_STATUS): void => { const setHealthStatus = (status: HEALTH_STATUS): void => {
healthStatus = status; healthStatus = status;

View File

@@ -6,7 +6,7 @@ import {
ZERO, ZERO,
QUOTE_PRECISION, QUOTE_PRECISION,
PRICE_PRECISION, PRICE_PRECISION,
BASE_PRECISION BASE_PRECISION,
} from '@drift-labs/sdk'; } from '@drift-labs/sdk';
import { import {
createMarketBasedAuctionParams, createMarketBasedAuctionParams,

View File

@@ -22,7 +22,6 @@ import {
AssetType, AssetType,
MainnetSpotMarkets, MainnetSpotMarkets,
DevnetSpotMarkets, DevnetSpotMarkets,
QUOTE_PRECISION,
PERCENTAGE_PRECISION_EXP, PERCENTAGE_PRECISION_EXP,
} from '@drift-labs/sdk'; } from '@drift-labs/sdk';
import { RedisClient } from '@drift/common/clients'; import { RedisClient } from '@drift/common/clients';