From 5a92750b54cae9d9ee76095b57402c59282ccc6c Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 20 Nov 2024 17:53:47 +0000 Subject: [PATCH 01/13] Bumping drift-common to 5b9a0026783451597eae8d7218a400338928a165 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 3917402..5b9a002 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 3917402307194fa6d992c1d749513cb89317c889 +Subproject commit 5b9a0026783451597eae8d7218a400338928a165 From 22090f9f3b9d56e9bc12a01da946fffc3babfe12 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 12:45:59 +0000 Subject: [PATCH 02/13] Bumping drift-common to 8178bc7515ef09c51290c1463bf7eba3c28de214 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 5b9a002..8178bc7 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 5b9a0026783451597eae8d7218a400338928a165 +Subproject commit 8178bc7515ef09c51290c1463bf7eba3c28de214 From 8952cc20e0aa59b72c84f8c6dd404f0711e616ee Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 16:31:19 +0000 Subject: [PATCH 03/13] Bumping drift-common to c251089a1a4f10ad21814358290dab2541a6eb66 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 8178bc7..c251089 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 8178bc7515ef09c51290c1463bf7eba3c28de214 +Subproject commit c251089a1a4f10ad21814358290dab2541a6eb66 From d43d2d2591af35b425850f527aa1ce649ce7ab39 Mon Sep 17 00:00:00 2001 From: Jack Waller Date: Thu, 21 Nov 2024 11:57:03 -0500 Subject: [PATCH 04/13] chore: update trades to helius --- src/publishers/tradesPublisher.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/publishers/tradesPublisher.ts b/src/publishers/tradesPublisher.ts index 28dab5b..a625bd4 100644 --- a/src/publishers/tradesPublisher.ts +++ b/src/publishers/tradesPublisher.ts @@ -36,7 +36,7 @@ setGlobalDispatcher( require('dotenv').config(); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; const commitHash = process.env.COMMIT; -const redisClientPrefix = RedisClientPrefix.DLOB; +const redisClientPrefix = RedisClientPrefix.DLOB_HELIUS; //@ts-ignore const sdkConfig = initialize({ env: process.env.ENV }); From 4b9b337a60cfaa2d94b20dc74a1fdfc16fa73185 Mon Sep 17 00:00:00 2001 From: Jack Waller Date: Thu, 21 Nov 2024 12:22:47 -0500 Subject: [PATCH 05/13] chore: update clients to be env vars --- src/index.ts | 10 ++++++++-- src/publishers/tradesPublisher.ts | 4 +++- src/serverLite.ts | 9 ++++++++- src/wsConnectionManager.ts | 4 +++- 4 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/index.ts b/src/index.ts index e275b8d..a4ebfed 100644 --- a/src/index.ts +++ b/src/index.ts @@ -62,8 +62,14 @@ setGlobalDispatcher( require('dotenv').config(); -// Reading in Redis env vars -const REDIS_CLIENTS = [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS]; +const envClients = []; +const clients = process.env.REDIS_CLIENT.trim() + .replace(/^\[|\]$/g, '') + .split(/\s*,\s*/); + +clients.forEach((client) => envClients.push(RedisClientPrefix[client])); + +const REDIS_CLIENTS = envClients.length ? envClients : [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS]; console.log('Redis Clients:', REDIS_CLIENTS); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; diff --git a/src/publishers/tradesPublisher.ts b/src/publishers/tradesPublisher.ts index a625bd4..3f75af3 100644 --- a/src/publishers/tradesPublisher.ts +++ b/src/publishers/tradesPublisher.ts @@ -36,7 +36,9 @@ setGlobalDispatcher( require('dotenv').config(); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; const commitHash = process.env.COMMIT; -const redisClientPrefix = RedisClientPrefix.DLOB_HELIUS; +const REDIS_CLIENT = process.env.REDIS_CLIENT || 'DLOB'; +console.log('Redis Clients:', REDIS_CLIENT); +const redisClientPrefix = RedisClientPrefix[REDIS_CLIENT]; //@ts-ignore const sdkConfig = initialize({ env: process.env.ENV }); diff --git a/src/serverLite.ts b/src/serverLite.ts index 7d65dbb..9ab8e30 100644 --- a/src/serverLite.ts +++ b/src/serverLite.ts @@ -38,7 +38,14 @@ setGlobalDispatcher( require('dotenv').config(); // Reading in Redis env vars -const REDIS_CLIENTS = [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS]; +const envClients = []; +const clients = process.env.REDIS_CLIENT.trim() + .replace(/^\[|\]$/g, '') + .split(/\s*,\s*/); + +clients.forEach((client) => envClients.push(RedisClientPrefix[client])); + +const REDIS_CLIENTS = envClients.length ? envClients : [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS]; console.log('Redis Clients:', REDIS_CLIENTS); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; diff --git a/src/wsConnectionManager.ts b/src/wsConnectionManager.ts index 194f045..bc7c3ef 100644 --- a/src/wsConnectionManager.ts +++ b/src/wsConnectionManager.ts @@ -34,7 +34,9 @@ const WS_PORT = process.env.WS_PORT || '3000'; console.log(`WS LISTENER PORT : ${WS_PORT}`); const MAX_BUFFERED_AMOUNT = 300000; -const CHANNEL_PREFIX = RedisClientPrefix.DLOB; +const REDIS_CLIENT = process.env.REDIS_CLIENT || 'DLOB'; +const CHANNEL_PREFIX = RedisClientPrefix[REDIS_CLIENT]; +console.log('Redis Clients:', REDIS_CLIENT); const CHANNEL_PREFIX_HELIUS = RedisClientPrefix.DLOB_HELIUS; const sanitiseChannelForClient = (channel: string | undefined): string => { From e4a78db77053d4f09d7af5c012bce3085e51315b Mon Sep 17 00:00:00 2001 From: Jack Waller Date: Thu, 21 Nov 2024 13:04:06 -0500 Subject: [PATCH 06/13] chore: null checks --- src/index.ts | 4 ++-- src/serverLite.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/index.ts b/src/index.ts index a4ebfed..4214c4b 100644 --- a/src/index.ts +++ b/src/index.ts @@ -63,11 +63,11 @@ setGlobalDispatcher( require('dotenv').config(); const envClients = []; -const clients = process.env.REDIS_CLIENT.trim() +const clients = process.env.REDIS_CLIENT?.trim() .replace(/^\[|\]$/g, '') .split(/\s*,\s*/); -clients.forEach((client) => envClients.push(RedisClientPrefix[client])); +clients?.forEach((client) => envClients.push(RedisClientPrefix[client])); const REDIS_CLIENTS = envClients.length ? envClients : [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS]; console.log('Redis Clients:', REDIS_CLIENTS); diff --git a/src/serverLite.ts b/src/serverLite.ts index 9ab8e30..d2e4024 100644 --- a/src/serverLite.ts +++ b/src/serverLite.ts @@ -39,11 +39,11 @@ require('dotenv').config(); // Reading in Redis env vars const envClients = []; -const clients = process.env.REDIS_CLIENT.trim() +const clients = process.env.REDIS_CLIENT?.trim() .replace(/^\[|\]$/g, '') .split(/\s*,\s*/); -clients.forEach((client) => envClients.push(RedisClientPrefix[client])); +clients?.forEach((client) => envClients.push(RedisClientPrefix[client])); const REDIS_CLIENTS = envClients.length ? envClients : [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS]; console.log('Redis Clients:', REDIS_CLIENTS); From 84000b8e43b28af6213012227ab34473efca90e5 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 21 Nov 2024 20:54:38 +0000 Subject: [PATCH 07/13] Bumping drift-common to e67d009f244055511f1423b30cd48f18f766afb6 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index c251089..e67d009 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit c251089a1a4f10ad21814358290dab2541a6eb66 +Subproject commit e67d009f244055511f1423b30cd48f18f766afb6 From 96008aa2702d2bcfd477174a260dad29ae0d59d4 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 22 Nov 2024 05:15:39 +0000 Subject: [PATCH 08/13] Bumping drift-common to 33005401904ed85fd88e07f9a1933e1c663a95d9 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index e67d009..3300540 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit e67d009f244055511f1423b30cd48f18f766afb6 +Subproject commit 33005401904ed85fd88e07f9a1933e1c663a95d9 From 5c98fc09cbf52f2fc833876ef2c03a02ab84d4f1 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 25 Nov 2024 22:44:45 +0000 Subject: [PATCH 09/13] Bumping drift-common to ef16c0c1728ec6d2320a002f346835b091785d7c --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 3300540..ef16c0c 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 33005401904ed85fd88e07f9a1933e1c663a95d9 +Subproject commit ef16c0c1728ec6d2320a002f346835b091785d7c From 791f7a3757f9bd00bde0b708ac8156f7f0b21f20 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Tue, 26 Nov 2024 21:26:00 +0000 Subject: [PATCH 10/13] Bumping drift-common to 70eac64f8585290a326c61c0bd2f81814e77eaa6 --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index ef16c0c..70eac64 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit ef16c0c1728ec6d2320a002f346835b091785d7c +Subproject commit 70eac64f8585290a326c61c0bd2f81814e77eaa6 From cc4adfa061fcbadea3a0de797e8a0a75f0f59114 Mon Sep 17 00:00:00 2001 From: GitHub Actions <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 27 Nov 2024 16:27:43 +0000 Subject: [PATCH 11/13] Bumping drift-common to 79d7d2f2fb10b4c211d1c2ac58484a43cf54404b --- drift-common | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drift-common b/drift-common index 70eac64..79d7d2f 160000 --- a/drift-common +++ b/drift-common @@ -1 +1 @@ -Subproject commit 70eac64f8585290a326c61c0bd2f81814e77eaa6 +Subproject commit 79d7d2f2fb10b4c211d1c2ac58484a43cf54404b From fa8fd24fd44eae775a515b117550a6f9253cb69c Mon Sep 17 00:00:00 2001 From: wphan Date: Wed, 27 Nov 2024 11:00:34 -0800 Subject: [PATCH 12/13] reduce dlob publisher health check grace period, clean up logs --- src/core/metrics.ts | 5 ++--- src/dlob-subscriber/DLOBSubscriberIO.ts | 14 ++++++-------- src/index.ts | 4 +++- src/publishers/dlobPublisher.ts | 4 ++-- src/serverLite.ts | 4 +++- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/src/core/metrics.ts b/src/core/metrics.ts index 3e1acb8..13d03d5 100644 --- a/src/core/metrics.ts +++ b/src/core/metrics.ts @@ -181,9 +181,7 @@ export const setHealthStatus = (status: HEALTH_STATUS): void => { * We may be hit by multiple sources performing health checks on us, so this middleware will latch * to its health state and only update every `healthCheckInterval`. * - * A grace period is also used to only report unhealthy if we have been unhealthy for a certain - * amount of time. This prevents reporting unhealthy even if we are just in the middle of a - * bulk account load. + * A grace period is also used to only report unhealthy if we have been unhealthy beyond the grace period. */ const handleHealthCheck = ( healthCheckGracePeriod: number, @@ -191,6 +189,7 @@ const handleHealthCheck = ( ) => { return async (_req, res, _next) => { if (healthStatus === HEALTH_STATUS.Restart) { + logger.error(`Health status: Restart`); res.writeHead(500); res.end(`NOK`); return; diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index 7d8a4aa..b5bf77a 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -221,10 +221,9 @@ export class DLOBSubscriberIO extends DLOBSubscriber { !skipSlotCheck && !isPerpMarketAndPrelaunchMarket ) { - console.log(`Unhealthy process due to slot diffs for market ${marketName}: - dlobProvider slot: ${slot} - oracle slot: ${l2Formatted['oracleData']['slot']} - `); + console.log( + `Unhealthy process due to slot diffs for market ${marketName}. dlobProviderSlot: ${slot}, oracleSlot: ${l2Formatted['oracleData']['slot']}` + ); setHealthStatus(HEALTH_STATUS.Restart); } @@ -239,10 +238,9 @@ export class DLOBSubscriberIO extends DLOBSubscriber { Date.now() - lastMarketSlotAndTime.ts > MAKRET_STALENESS_THRESHOLD && !skipSlotCheck ) { - console.log(`Unhealthy process due to same slot for market ${marketName} after > ${MAKRET_STALENESS_THRESHOLD}ms: - dlobProvider slot: ${slot} - market slot: ${l2Formatted['marketSlot']} - `); + console.log( + `Unhealthy process due to same slot for market ${marketName} after > ${MAKRET_STALENESS_THRESHOLD}ms. dlobProviderSlot: ${slot}, marketSlot: ${l2Formatted['marketSlot']}` + ); setHealthStatus(HEALTH_STATUS.Restart); } else if ( lastMarketSlotAndTime && diff --git a/src/index.ts b/src/index.ts index 4214c4b..950daef 100644 --- a/src/index.ts +++ b/src/index.ts @@ -69,7 +69,9 @@ const clients = process.env.REDIS_CLIENT?.trim() clients?.forEach((client) => envClients.push(RedisClientPrefix[client])); -const REDIS_CLIENTS = envClients.length ? envClients : [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS]; +const REDIS_CLIENTS = envClients.length + ? envClients + : [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS]; console.log('Redis Clients:', REDIS_CLIENTS); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index b79af54..c5f5037 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -455,10 +455,10 @@ const main = async () => { app.get( '/health', - handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, dlobProvider) + handleHealthCheck(WS_FALLBACK_FETCH_INTERVAL, dlobProvider) ); app.get('/startup', handleStartup); - app.get('/', handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, dlobProvider)); + app.get('/', handleHealthCheck(WS_FALLBACK_FETCH_INTERVAL, dlobProvider)); const server = app.listen(8080); // Default keepalive is 5s, since the AWS ALB timeout is 60 seconds, clients diff --git a/src/serverLite.ts b/src/serverLite.ts index d2e4024..f79044c 100644 --- a/src/serverLite.ts +++ b/src/serverLite.ts @@ -45,7 +45,9 @@ const clients = process.env.REDIS_CLIENT?.trim() clients?.forEach((client) => envClients.push(RedisClientPrefix[client])); -const REDIS_CLIENTS = envClients.length ? envClients : [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS]; +const REDIS_CLIENTS = envClients.length + ? envClients + : [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS]; console.log('Redis Clients:', REDIS_CLIENTS); const driftEnv = (process.env.ENV || 'devnet') as DriftEnv; From 00e167da191548b86838ed572b010d29ba11ce33 Mon Sep 17 00:00:00 2001 From: wphan Date: Wed, 27 Nov 2024 11:10:34 -0800 Subject: [PATCH 13/13] fix logs --- src/core/metrics.ts | 6 +++--- src/dlob-subscriber/DLOBSubscriberIO.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/core/metrics.ts b/src/core/metrics.ts index 13d03d5..bd6b883 100644 --- a/src/core/metrics.ts +++ b/src/core/metrics.ts @@ -181,7 +181,7 @@ export const setHealthStatus = (status: HEALTH_STATUS): void => { * We may be hit by multiple sources performing health checks on us, so this middleware will latch * to its health state and only update every `healthCheckInterval`. * - * A grace period is also used to only report unhealthy if we have been unhealthy beyond the grace period. + * A grace period is also used to report unhealthy only if we have been unhealthy beyond the grace period. */ const handleHealthCheck = ( healthCheckGracePeriod: number, @@ -226,14 +226,14 @@ const handleHealthCheck = ( lastHealthCheckPerformed = Date.now(); if (!lastHealthCheckState && !inGracePeriod) { - healthStatus = HEALTH_STATUS.UnhealthySlotSubscriber; + setHealthStatus(HEALTH_STATUS.UnhealthySlotSubscriber); res.writeHead(500); res.end(`NOK`); return; } - healthStatus = HEALTH_STATUS.Ok; + setHealthStatus(HEALTH_STATUS.Ok); res.writeHead(200); res.end('OK'); }; diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index b5bf77a..0a603f9 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -238,7 +238,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { Date.now() - lastMarketSlotAndTime.ts > MAKRET_STALENESS_THRESHOLD && !skipSlotCheck ) { - console.log( + logger.warn( `Unhealthy process due to same slot for market ${marketName} after > ${MAKRET_STALENESS_THRESHOLD}ms. dlobProviderSlot: ${slot}, marketSlot: ${l2Formatted['marketSlot']}` ); setHealthStatus(HEALTH_STATUS.Restart); @@ -246,7 +246,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { lastMarketSlotAndTime && l2Formatted['marketSlot'] !== lastMarketSlotAndTime.slot ) { - console.log( + logger.warn( `Updating market slot for ${marketArgs.marketName} with slot ${l2Formatted['marketSlot']}` ); this.lastMarketSlotMap