trade/trade-drift-dlob
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add grace period to health check

Browse Source
This commit is contained in:
wphan
2023-12-02 14:02:44 -08:00
parent 0ce6cb4938
commit 12281198e5
2 changed files with 21 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
src/core/metrics.ts
View File

@@ -135,14 +135,23 @@ const healthCheckInterval = 2 * (ORDERBOOK_UPDATE_INTERVAL ?? 1000); // ORDERBOO
let lastHealthCheckSlot = -1; let lastHealthCheckSlot = -1;
let lastHealthCheckState = true; // true = healthy, false = unhealthy let lastHealthCheckState = true; // true = healthy, false = unhealthy
let lastHealthCheckPerformed = Date.now() - healthCheckInterval; let lastHealthCheckPerformed = Date.now() - healthCheckInterval;
let lastTimeHealthy = Date.now() - healthCheckInterval;
/** /**
* Middleware that checks if we are in general healthy by checking that the bulk account loader slot * Middleware that checks if we are in general healthy by checking that the bulk account loader slot
* has changed recently. * has changed recently.
* *
* We may be hit by multiple sources performing health checks on us, so this middleware will latch * We may be hit by multiple sources performing health checks on us, so this middleware will latch
* to its health state and only update every `healthCheckInterval`. * to its health state and only update every `healthCheckInterval`.
*
* A grace period is also used to only report unhealthy if we have been unhealthy for a certain
* amount of time. This prevents reporting unhealthy even if we are just in the middle of a
* bulk account load.
*/ */
const handleHealthCheck = (slotSource: SlotSource) => { const handleHealthCheck = (
healthCheckGracePeriod: number,
slotSource: SlotSource
) => {
return async (_req, res, _next) => { return async (_req, res, _next) => {
if (Date.now() < lastHealthCheckPerformed + healthCheckInterval) { if (Date.now() < lastHealthCheckPerformed + healthCheckInterval) {
if (lastHealthCheckState) { if (lastHealthCheckState) {
@@ -163,12 +172,17 @@ const handleHealthCheck = (slotSource: SlotSource) => {
Date.now() - lastHealthCheckPerformed Date.now() - lastHealthCheckPerformed
} ms` } ms`
); );
} else {
lastTimeHealthy = Date.now();
} }
lastHealthCheckSlot = lastSlotReceived; lastHealthCheckSlot = lastSlotReceived;
lastHealthCheckPerformed = Date.now(); lastHealthCheckPerformed = Date.now();
if (!lastHealthCheckState) { if (
!lastHealthCheckState &&
Date.now() - lastTimeHealthy > healthCheckGracePeriod
) {
healthStatus = HEALTH_STATUS.UnhealthySlotSubscriber; healthStatus = HEALTH_STATUS.UnhealthySlotSubscriber;
res.writeHead(500); res.writeHead(500);

7
src/index.ts
View File

@@ -331,9 +331,12 @@ const main = async () => {
} }
}; };
app.get('/health', handleHealthCheck(slotSource)); app.get(
'/health',
handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, slotSource)
);
app.get('/startup', handleStartup); app.get('/startup', handleStartup);
app.get('/', handleHealthCheck(slotSource)); app.get('/', handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, slotSource));
if (FEATURE_FLAGS.ENABLE_ORDERS_ENDPOINTS) { if (FEATURE_FLAGS.ENABLE_ORDERS_ENDPOINTS) {
app.get('/orders/json/raw', async (_req, res, next) => { app.get('/orders/json/raw', async (_req, res, next) => {