From 10dc70b6acba32747b2c3450fdf2d0036167897f Mon Sep 17 00:00:00 2001 From: wphan Date: Fri, 27 Oct 2023 12:19:12 -0700 Subject: [PATCH 1/8] bump skd --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index cde8518..9e7c4ea 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.42.0-beta.14", + "@drift-labs/sdk": "2.43.0-beta.1", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 38e158f..02b188e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.42.0-beta.14": - version "2.42.0-beta.14" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.42.0-beta.14.tgz#b29976ca4bcfdeab74191369f6197faece8d4b90" - integrity sha512-C3J3VDoXkWrk8S6z1zO4NNm7kZoqhAKMROD6WM4Yn4dKY/jDq6KwfjlVnZ7iwxsO1YTqoLP/N9N29JsLfHVqgg== +"@drift-labs/sdk@2.43.0-beta.1": + version "2.43.0-beta.1" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.1.tgz#b1cc01fa5f1bdba6c4597b783f75cc24cf74909e" + integrity sha512-Ij0+C2d3bhoLhAMNqy4omqe8tN+CSAAOVUrgVCBNFr9LHwmUFITJLt11vpj355FdJxH1frR7EW2s4M3lZdsrog== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 585ebcb5faec47e159bc9b9c08d8555a70aef06b Mon Sep 17 00:00:00 2001 From: wphan Date: Fri, 3 Nov 2023 11:19:27 -0700 Subject: [PATCH 2/8] add slot to dlob server --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 9e7c4ea..7262a95 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.43.0-beta.1", + "@drift-labs/sdk": "2.43.0-beta.7", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 02b188e..62d9c11 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.43.0-beta.1": - version "2.43.0-beta.1" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.1.tgz#b1cc01fa5f1bdba6c4597b783f75cc24cf74909e" - integrity sha512-Ij0+C2d3bhoLhAMNqy4omqe8tN+CSAAOVUrgVCBNFr9LHwmUFITJLt11vpj355FdJxH1frR7EW2s4M3lZdsrog== +"@drift-labs/sdk@2.43.0-beta.7": + version "2.43.0-beta.7" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.7.tgz#b93b7f186d3588cdf85af081414abb028cf3c928" + integrity sha512-XZTC/twPQOyB6kT5TTnC7VO8avjwiSy8DKSWvcFKB9eB+o01qlZuNE6bhUeY+ZnPIyk5Rd6bLVKVPH1umw+G4Q== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 74e6da26937a63cff4ee9fd4007dd95a59f9bf45 Mon Sep 17 00:00:00 2001 From: Chris Heaney Date: Tue, 7 Nov 2023 15:26:45 +0100 Subject: [PATCH 3/8] update sdk --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 7262a95..087c5a2 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.43.0-beta.7", + "@drift-labs/sdk": "2.43.0-beta.10", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 62d9c11..6e4947e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.43.0-beta.7": - version "2.43.0-beta.7" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.7.tgz#b93b7f186d3588cdf85af081414abb028cf3c928" - integrity sha512-XZTC/twPQOyB6kT5TTnC7VO8avjwiSy8DKSWvcFKB9eB+o01qlZuNE6bhUeY+ZnPIyk5Rd6bLVKVPH1umw+G4Q== +"@drift-labs/sdk@2.43.0-beta.10": + version "2.43.0-beta.10" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.43.0-beta.10.tgz#afe05e7b6aaf557d495a2e4743767be4b6c27435" + integrity sha512-NqQ2jV7ewiAkL2nNdxnoi1C+tefnxskFo9rHtl6yUF6NpygE08usOcbCu5to+Cyrtx6cSAymHsxCrDU1c7lkQw== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 3ea8c1b9d341f0a843ff37f16feb5e344f784da5 Mon Sep 17 00:00:00 2001 From: Chris Heaney Date: Mon, 20 Nov 2023 10:30:02 -0500 Subject: [PATCH 4/8] update sdk --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 5b51040..60be382 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.45.0-beta.0", + "@drift-labs/sdk": "2.45.0-beta.1", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 5e3a7e9..e385363 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.45.0-beta.0": - version "2.45.0-beta.0" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.45.0-beta.0.tgz#9805d09bd44cddfdfe85c2f0af38fc963604fe9e" - integrity sha512-hZht2wAxv/QVh1f7cneY6e0dsV2d23hsupZXpkC9j6aepW74SyHQRu+Btjfjhh5w27ViqnAVv3Ab1O1hYzrJyw== +"@drift-labs/sdk@2.45.0-beta.1": + version "2.45.0-beta.1" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.45.0-beta.1.tgz#f1ebd950eb273a0649e4ce2eacd04b6d7f7fc28a" + integrity sha512-Ab05szcVv67jlJ3DgtY5UC2g+dG9AWTHtiS33Z2kMstf3OdgVp+wKFLL9Y9/xzpBI8nZo01QzsUU2P1SAi8N2A== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 56977a81d667728baadd4f08703f7dace6513ffd Mon Sep 17 00:00:00 2001 From: Chris Heaney Date: Tue, 21 Nov 2023 08:47:51 -0500 Subject: [PATCH 5/8] update sdk --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 60be382..e3d2fde 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.45.0-beta.1", + "@drift-labs/sdk": "2.45.0-beta.4", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index e385363..88549ca 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.45.0-beta.1": - version "2.45.0-beta.1" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.45.0-beta.1.tgz#f1ebd950eb273a0649e4ce2eacd04b6d7f7fc28a" - integrity sha512-Ab05szcVv67jlJ3DgtY5UC2g+dG9AWTHtiS33Z2kMstf3OdgVp+wKFLL9Y9/xzpBI8nZo01QzsUU2P1SAi8N2A== +"@drift-labs/sdk@2.45.0-beta.4": + version "2.45.0-beta.4" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.45.0-beta.4.tgz#237d5d1fd55fef60964ab359536fc03ff757eedd" + integrity sha512-NNbWXEnFyL99J0fyByV20gtsN0DnwAz3hoscVPWi+ejqhEqgfcuc1BVqtLX1lwSiWsxDQnIMGz437HSuFimaVQ== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From 1e3ebb457615e650e95064c3a5581a2d89da7be4 Mon Sep 17 00:00:00 2001 From: Chris Heaney Date: Wed, 22 Nov 2023 17:39:24 -0500 Subject: [PATCH 6/8] update sdk --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index e3d2fde..d35a628 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "main": "lib/index.js", "license": "Apache-2.0", "dependencies": { - "@drift-labs/sdk": "2.45.0-beta.4", + "@drift-labs/sdk": "2.46.0-beta.1", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/yarn.lock b/yarn.lock index 88549ca..3fa51b3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -80,10 +80,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.45.0-beta.4": - version "2.45.0-beta.4" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.45.0-beta.4.tgz#237d5d1fd55fef60964ab359536fc03ff757eedd" - integrity sha512-NNbWXEnFyL99J0fyByV20gtsN0DnwAz3hoscVPWi+ejqhEqgfcuc1BVqtLX1lwSiWsxDQnIMGz437HSuFimaVQ== +"@drift-labs/sdk@2.46.0-beta.1": + version "2.46.0-beta.1" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.46.0-beta.1.tgz#0fea7be51a07180479418ee559fa916ee306db9b" + integrity sha512-r5edKj5UlZPn6ChWab9vlBWVeHQayszF/RdLr/IU4+iAYcT0VYSbkI53WmB86z6ox07uBK9VGvtPQlfKtrVh0A== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" From f1504a7404fd33b8ba025e8df0f2805e706dc18b Mon Sep 17 00:00:00 2001 From: wphan Date: Fri, 24 Nov 2023 14:20:58 -0800 Subject: [PATCH 7/8] increase keepAliveTimeout --- src/index.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/index.ts b/src/index.ts index 8ff170f..0a8c235 100644 --- a/src/index.ts +++ b/src/index.ts @@ -12,7 +12,6 @@ import { DriftClient, initialize, DriftEnv, - SlotSubscriber, UserMap, DLOBOrder, DLOBOrders, @@ -104,6 +103,13 @@ app.use((req, _res, next) => { app.use(errorHandler); const server = http.createServer(app); +// Default keepalive is 5s, since the AWS ALB timeout is 60 seconds, clients +// sometimes get 502s. +// https://shuheikagawa.com/blog/2019/04/25/keep-alive-timeout/ +// https://stackoverflow.com/a/68922692 +server.keepAliveTimeout = 61 * 1000; +server.headersTimeout = 65 * 1000; + const opts = program.opts(); setLogLevel(opts.debug ? 'debug' : 'info'); @@ -212,11 +218,7 @@ const main = async () => { await userMap.subscribe(); const userStatsMap = new UserStatsMap(driftClient, { type: 'polling', - accountLoader: new BulkAccountLoader( - connection, - stateCommitment, - 0 - ), + accountLoader: new BulkAccountLoader(connection, stateCommitment, 0), }); await userStatsMap.subscribe(); From aefb235cea30f0e3ce52cec8b022869f2dece136 Mon Sep 17 00:00:00 2001 From: wphan Date: Fri, 24 Nov 2023 16:15:11 -0800 Subject: [PATCH 8/8] debug unhealthy SlotSubscriber --- src/core/metrics.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/core/metrics.ts b/src/core/metrics.ts index 6032d9e..bdeb495 100644 --- a/src/core/metrics.ts +++ b/src/core/metrics.ts @@ -126,9 +126,11 @@ const handleHealthCheck = async (req, res, next) => { if (req.url === '/health' || req.url === '/') { // check if a slot was received recently let healthySlotSubscriber = false; + let slotChanged = false; + let slotChangedRecently = false; await lastSlotReceivedMutex.runExclusive(async () => { - const slotChanged = lastSlotReceived > lastHealthCheckSlot; - const slotChangedRecently = + slotChanged = lastSlotReceived > lastHealthCheckSlot; + slotChangedRecently = Date.now() - lastHealthCheckSlotUpdated < healthCheckInterval; healthySlotSubscriber = slotChanged || slotChangedRecently; logger.debug( @@ -142,6 +144,9 @@ const handleHealthCheck = async (req, res, next) => { if (!healthySlotSubscriber) { healthStatus = HEALTH_STATUS.UnhealthySlotSubscriber; logger.error(`SlotSubscriber is not healthy`); + logger.error( + `Slotsubscriber health check: lastSlotReceived: ${lastSlotReceived}, lastHealthCheckSlot: ${lastHealthCheckSlot}, slotChanged: ${slotChanged}, slotChangedRecently: ${slotChangedRecently}` + ); res.writeHead(500); res.end(`SlotSubscriber is not healthy`);