diff --git a/Dockerfile b/Dockerfile index 37e7284..d9e3758 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM public.ecr.aws/bitnami/node:16 +FROM public.ecr.aws/bitnami/node:18 RUN apt-get install git ENV NODE_ENV=production RUN npm install -g yarn @@ -9,6 +9,4 @@ COPY . . RUN yarn RUN yarn build -EXPOSE 9464 - -CMD [ "yarn", "start:all" ] \ No newline at end of file +EXPOSE 9464 \ No newline at end of file diff --git a/package.json b/package.json index 760aae1..90365b5 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "license": "Apache-2.0", "dependencies": { "@coral-xyz/anchor": "^0.29.0", - "@drift-labs/sdk": "2.74.0-beta.10", + "@drift-labs/sdk": "2.76.0-beta.1", "@opentelemetry/api": "^1.1.0", "@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/exporter-prometheus": "^0.31.0", diff --git a/src/core/metrics.ts b/src/core/metrics.ts index 574c245..3e1acb8 100644 --- a/src/core/metrics.ts +++ b/src/core/metrics.ts @@ -45,6 +45,7 @@ export enum HEALTH_STATUS { StaleBulkAccountLoader, UnhealthySlotSubscriber, LivenessTesting, + Restart, } const metricsPort = @@ -169,6 +170,10 @@ let lastHealthCheckState = true; // true = healthy, false = unhealthy let lastHealthCheckPerformed = Date.now() - healthCheckInterval; let lastTimeHealthy = Date.now() - healthCheckInterval; +export const setHealthStatus = (status: HEALTH_STATUS): void => { + healthStatus = status; +}; + /** * Middleware that checks if we are in general healthy by checking that the bulk account loader slot * has changed recently. @@ -185,6 +190,12 @@ const handleHealthCheck = ( slotSource: SlotSource ) => { return async (_req, res, _next) => { + if (healthStatus === HEALTH_STATUS.Restart) { + res.writeHead(500); + res.end(`NOK`); + return; + } + if (Date.now() < lastHealthCheckPerformed + healthCheckInterval) { if (lastHealthCheckState) { res.writeHead(200); diff --git a/src/dlob-subscriber/DLOBSubscriberIO.ts b/src/dlob-subscriber/DLOBSubscriberIO.ts index f8291fd..51857db 100644 --- a/src/dlob-subscriber/DLOBSubscriberIO.ts +++ b/src/dlob-subscriber/DLOBSubscriberIO.ts @@ -16,6 +16,7 @@ import { addOracletoResponse, l2WithBNToStrings, } from '../utils/utils'; +import { setHealthStatus, HEALTH_STATUS } from '../core/metrics'; type wsMarketArgs = { marketIndex: number; @@ -34,7 +35,7 @@ const SPOT_MAKRET_STALENESS_THRESHOLD = 20 * 60 * 1000; const PERP_MARKETS_TO_SKIP_SLOT_CHECK = { 'mainnet-beta': [17], - devnet: [17, 21], + devnet: [17, 21, 23], }; const SPOT_MARKETS_TO_SKIP_SLOT_CHECK = { @@ -62,7 +63,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber { constructor( config: DLOBSubscriptionConfig & { - env: DriftEnv, + env: DriftEnv; redisClient: RedisClient; perpMarketInfos: wsMarketInfo[]; spotMarketInfos: wsMarketInfo[]; @@ -209,11 +210,11 @@ export class DLOBSubscriberIO extends DLOBSubscriber { this.killSwitchSlotDiffThreshold && !skipSlotCheck ) { - console.log(`Killing process due to slot diffs for market ${marketName}: + console.log(`Unhealthy process due to slot diffs for market ${marketName}: dlobProvider slot: ${slot} oracle slot: ${l2Formatted['oracleData']['slot']} `); - process.exit(1); + setHealthStatus(HEALTH_STATUS.Restart); } // Check if times and slots are too different for market @@ -227,11 +228,11 @@ export class DLOBSubscriberIO extends DLOBSubscriber { Date.now() - lastMarketSlotAndTime.ts > MAKRET_STALENESS_THRESHOLD && !skipSlotCheck ) { - console.log(`Killing process due to same slot for market ${marketName} after > ${MAKRET_STALENESS_THRESHOLD}ms: + console.log(`Unhealthy process due to same slot for market ${marketName} after > ${MAKRET_STALENESS_THRESHOLD}ms: dlobProvider slot: ${slot} market slot: ${l2Formatted['marketSlot']} `); - process.exit(1); + setHealthStatus(HEALTH_STATUS.Restart); } else if ( lastMarketSlotAndTime && l2Formatted['marketSlot'] !== lastMarketSlotAndTime.slot diff --git a/src/index.ts b/src/index.ts index e88326d..489fa49 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,7 +2,6 @@ import { program } from 'commander'; import compression from 'compression'; import cors from 'cors'; import express from 'express'; -import rateLimit from 'express-rate-limit'; import morgan from 'morgan'; import { Commitment, Connection, Keypair, PublicKey } from '@solana/web3.js'; @@ -99,10 +98,6 @@ const SLOT_STALENESS_TOLERANCE = parseInt(process.env.SLOT_STALENESS_TOLERANCE) || 20; const ROTATION_COOLDOWN = parseInt(process.env.ROTATION_COOLDOWN) || 5000; const useWebsocket = process.env.USE_WEBSOCKET?.toLowerCase() === 'true'; -const rateLimitCallsPerSecond = process.env.RATE_LIMIT_CALLS_PER_SECOND - ? parseInt(process.env.RATE_LIMIT_CALLS_PER_SECOND) - : 1; -const loadTestAllowed = process.env.ALLOW_LOAD_TEST?.toLowerCase() === 'true'; const useRedis = process.env.USE_REDIS?.toLowerCase() === 'true'; const logFormat = @@ -119,23 +114,6 @@ app.use(compression()); app.set('trust proxy', 1); app.use(logHttp); app.use(handleResponseTime); -if (!FEATURE_FLAGS.DISABLE_RATE_LIMIT) { - app.use( - rateLimit({ - windowMs: 1000, // 1 second - max: rateLimitCallsPerSecond, - standardHeaders: false, // Return rate limit info in the `RateLimit-*` headers - legacyHeaders: false, // Disable the `X-RateLimit-*` headers - skip: (req, _res) => { - if (!loadTestAllowed) { - return false; - } - - return req.headers['user-agent'].includes('k6'); - }, - }) - ); -} // strip off /dlob, if the request comes from exchange history server LB app.use((req, _res, next) => { diff --git a/src/publishers/dlobPublisher.ts b/src/publishers/dlobPublisher.ts index 659299f..2be39bd 100644 --- a/src/publishers/dlobPublisher.ts +++ b/src/publishers/dlobPublisher.ts @@ -40,6 +40,8 @@ import { } from '../dlobProvider'; import FEATURE_FLAGS from '../utils/featureFlags'; import { GeyserOrderSubscriber } from '../grpc/OrderSubscriberGRPC'; +import express from 'express'; +import { handleHealthCheck } from '../core/metrics'; require('dotenv').config(); const stateCommitment: Commitment = 'confirmed'; @@ -49,6 +51,9 @@ const REDIS_HOST = process.env.REDIS_HOST || 'localhost'; const REDIS_PORT = process.env.REDIS_PORT || '6379'; const REDIS_PASSWORD = process.env.REDIS_PASSWORD; +// Set up express for health checks +const app = express(); + //@ts-ignore const sdkConfig = initialize({ env: process.env.ENV }); let driftClient: DriftClient; @@ -420,6 +425,31 @@ const main = async () => { recursiveFetch(); } + const handleStartup = async (_req, res, _next) => { + if (driftClient.isSubscribed && dlobProvider.size() > 0) { + res.writeHead(200); + res.end('OK'); + } else { + res.writeHead(500); + res.end('Not ready'); + } + }; + + app.get( + '/health', + handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, dlobProvider) + ); + app.get('/startup', handleStartup); + app.get('/', handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, dlobProvider)); + const server = app.listen(8080); + + // Default keepalive is 5s, since the AWS ALB timeout is 60 seconds, clients + // sometimes get 502s. + // https://shuheikagawa.com/blog/2019/04/25/keep-alive-timeout/ + // https://stackoverflow.com/a/68922692 + server.keepAliveTimeout = 61 * 1000; + server.headersTimeout = 65 * 1000; + console.log('DLOBSubscriber Publishing Messages'); }; diff --git a/yarn.lock b/yarn.lock index 37b7b4a..3855683 100644 --- a/yarn.lock +++ b/yarn.lock @@ -115,10 +115,10 @@ enabled "2.0.x" kuler "^2.0.0" -"@drift-labs/sdk@2.74.0-beta.10": - version "2.74.0-beta.10" - resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.74.0-beta.10.tgz#1a3130cb74293a58d04fd66a87ee54c756e9c79a" - integrity sha512-INyLJhcPHL7W13A5EOAXm323f4lVYEzwgXHaBV1lzqU0H0m2/7JHquxgkq5+BKV+gkL8xb+aglRKWTs2qkAhMQ== +"@drift-labs/sdk@2.76.0-beta.1": + version "2.76.0-beta.1" + resolved "https://registry.yarnpkg.com/@drift-labs/sdk/-/sdk-2.76.0-beta.1.tgz#d483e0ce5938a9fd81532a0ff23a49fe939710f2" + integrity sha512-NAtfLcp6tbDaR5cXaPsYvqSqclfjFo3GrvupzMImbz2b0ilgeeToW+JwHRlLgcOZaj+9JG7ECi31kHaHmItf9w== dependencies: "@coral-xyz/anchor" "0.28.1-beta.2" "@ellipsis-labs/phoenix-sdk" "^1.4.2" @@ -128,6 +128,7 @@ "@solana/web3.js" "1.73.2" strict-event-emitter-types "^2.0.0" uuid "^8.3.2" + zstddec "^0.1.0" "@ellipsis-labs/phoenix-sdk@^1.4.2": version "1.4.2" @@ -4507,3 +4508,8 @@ yn@3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/yn/-/yn-3.1.1.tgz#1e87401a09d767c1d5eab26a6e4c185182d2eb50" integrity sha512-Ux4ygGWsu2c7isFWe8Yu1YluJmqVhxqK2cLXNQA5AcC3QfbGNpM7fu0Y8b/z16pXLnFxZYvWhd3fhBY9DLmC6Q== + +zstddec@^0.1.0: + version "0.1.0" + resolved "https://registry.yarnpkg.com/zstddec/-/zstddec-0.1.0.tgz#7050f3f0e0c3978562d0c566b3e5a427d2bad7ec" + integrity sha512-w2NTI8+3l3eeltKAdK8QpiLo/flRAr2p8AGeakfMZOXBxOg9HIu4LVDxBi81sYgVhFhdJjv1OrB5ssI8uFPoLg==