revamp server lite

This commit is contained in:
Nour Alharithi
2024-07-26 15:29:16 -07:00
parent 8924f66156
commit 3de6dc8f55
3 changed files with 211 additions and 411 deletions

View File

@@ -144,7 +144,7 @@ export class DLOBSubscriberIO extends DLOBSubscriber {
: this.driftClient.getOracleDataForSpotMarket(marketArgs.marketIndex)
.slot;
let includeVamm = marketArgs.includeVamm;
if (dlobSlot - oracleSlot > STALE_ORACLE_REMOVE_VAMM_THRESHOLD) {
if (dlobSlot - oracleSlot.toNumber() > STALE_ORACLE_REMOVE_VAMM_THRESHOLD) {
logger.info('Oracle is stale, removing vamm orders');
includeVamm = false;
}

View File

@@ -682,7 +682,9 @@ const main = async (): Promise<void> => {
if (useRedis) {
if (!isSpot && `${includeVamm}`?.toLowerCase() === 'true') {
const redisClient = perpMarketRedisMap.get(normedMarketIndex).client;
const redisL2 = await redisClient.get(`last_update_orderbook_perp_${normedMarketIndex}`);
const redisL2 = await redisClient.get(
`last_update_orderbook_perp_${normedMarketIndex}`
);
const depth = Math.min(parseInt(adjustedDepth as string) ?? 1, 100);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
@@ -706,7 +708,9 @@ const main = async (): Promise<void> => {
`${includePhoenix}`?.toLowerCase() === 'true'
) {
const redisClient = spotMarketRedisMap.get(normedMarketIndex).client;
const redisL2 = await redisClient.get(`last_update_orderbook_spot_${normedMarketIndex}`);
const redisL2 = await redisClient.get(
`last_update_orderbook_spot_${normedMarketIndex}`
);
const depth = Math.min(parseInt(adjustedDepth as string) ?? 1, 100);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
@@ -836,8 +840,13 @@ const main = async (): Promise<void> => {
) {
const redisClient =
perpMarketRedisMap.get(normedMarketIndex).client;
const redisL2 = await redisClient.get(`last_update_orderbook_perp_${normedMarketIndex}`);
const depth = Math.min(parseInt(adjustedDepth as string) ?? 1, 100);
const redisL2 = await redisClient.get(
`last_update_orderbook_perp_${normedMarketIndex}`
);
const depth = Math.min(
parseInt(adjustedDepth as string) ?? 1,
100
);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
if (redisL2) {
@@ -861,8 +870,13 @@ const main = async (): Promise<void> => {
) {
const redisClient =
spotMarketRedisMap.get(normedMarketIndex).client;
const redisL2 = await redisClient.get(`last_update_orderbook_spot_${normedMarketIndex}`);
const depth = Math.min(parseInt(adjustedDepth as string) ?? 1, 100);
const redisL2 = await redisClient.get(
`last_update_orderbook_spot_${normedMarketIndex}`
);
const depth = Math.min(
parseInt(adjustedDepth as string) ?? 1,
100
);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
if (redisL2) {

View File

@@ -6,26 +6,33 @@ import morgan from 'morgan';
import { Commitment, Connection } from '@solana/web3.js';
import {
DriftEnv,
SlotSubscriber,
initialize,
isVariant,
MarketType,
} from '@drift-labs/sdk';
import { RedisClient, RedisClientPrefix } from '@drift/common';
import { logger, setLogLevel } from './utils/logger';
import * as http from 'http';
import {
handleHealthCheck,
cacheHitCounter,
incomingRequestsCounter,
runtimeSpecsGauge,
} from './core/metrics';
import { handleResponseTime } from './core/middleware';
import { errorHandler, normalizeBatchQueryParams, sleep } from './utils/utils';
import {
DriftEnv,
MarketType,
PerpMarkets,
SlotSubscriber,
SpotMarkets,
isVariant,
initialize,
} from '@drift-labs/sdk';
import { RedisClient, RedisClientPrefix } from '@drift/common';
import { errorHandler, sleep } from './utils/utils';
import { setGlobalDispatcher, Agent } from 'undici';
setGlobalDispatcher(
new Agent({
connections: 200,
})
);
require('dotenv').config();
@@ -38,11 +45,15 @@ const commitHash = process.env.COMMIT;
//@ts-ignore
const sdkConfig = initialize({ env: process.env.ENV });
const stateCommitment: Commitment = 'processed';
const stateCommitment: Commitment = 'confirmed';
const serverPort = process.env.PORT || 6969;
export const ORDERBOOK_UPDATE_INTERVAL = 1000;
const WS_FALLBACK_FETCH_INTERVAL = ORDERBOOK_UPDATE_INTERVAL * 60;
const SLOT_STALENESS_TOLERANCE =
parseInt(process.env.SLOT_STALENESS_TOLERANCE) || 20;
parseInt(process.env.SLOT_STALENESS_TOLERANCE) || 35;
const ROTATION_COOLDOWN = parseInt(process.env.ROTATION_COOLDOWN) || 5000;
const useWebsocket = process.env.USE_WEBSOCKET?.toLowerCase() === 'true';
const useRedis = process.env.USE_REDIS?.toLowerCase() === 'true';
const logFormat =
':remote-addr - :remote-user [:date[clf]] ":method :url HTTP/:http-version" :status :res[content-length] ":referrer" ":user-agent" :req[x-forwarded-for]';
@@ -57,6 +68,18 @@ app.set('trust proxy', 1);
app.use(logHttp);
app.use(handleResponseTime);
// strip off /dlob, if the request comes from exchange history server LB
app.use((req, _res, next) => {
if (req.url.startsWith('/dlob')) {
req.url = req.url.replace('/dlob', '');
if (req.url === '') {
req.url = '/';
}
}
incomingRequestsCounter.add(1);
next();
});
// Metrics defined here
const bootTimeMs = Date.now();
runtimeSpecsGauge.addCallback((obs) => {
@@ -68,28 +91,6 @@ runtimeSpecsGauge.addCallback((obs) => {
});
});
// strip off /dlob, if the request comes from exchange history server LB
app.use((req, _res, next) => {
if (req.url.startsWith('/dlob')) {
req.url = req.url.replace('/dlob', '');
if (req.url === '') {
req.url = '/';
}
}
next();
});
// strip off /ui, if the request comes from the UI
app.use((req, _res, next) => {
if (req.url.startsWith('/ui')) {
req.url = req.url.replace('/ui', '');
if (req.url === '') {
req.url = '/';
}
}
next();
});
app.use(errorHandler);
const server = http.createServer(app);
@@ -107,63 +108,120 @@ const endpoint = process.env.ENDPOINT;
const wsEndpoint = process.env.WS_ENDPOINT;
logger.info(`RPC endpoint: ${endpoint}`);
logger.info(`WS endpoint: ${wsEndpoint}`);
logger.info(`useWebsocket: ${useWebsocket}`);
logger.info(`DriftEnv: ${driftEnv}`);
logger.info(`Commit: ${commitHash}`);
const main = async () => {
// Redis connect
const main = async (): Promise<void> => {
const connection = new Connection(endpoint, {
wsEndpoint,
commitment: stateCommitment,
});
const slotSubscriber = new SlotSubscriber(connection, {
resubTimeoutMs: 5000,
});
await slotSubscriber.subscribe();
// Handle redis client initialization and rotation maps
const redisClients: Array<RedisClient> = [];
const spotMarketRedisMap: Map<
number,
{ client: RedisClient; clientIndex: number }
{
client: RedisClient;
clientIndex: number;
lastRotationTime: number;
lock: boolean;
}
> = new Map();
const perpMarketRedisMap: Map<
number,
{ client: RedisClient; clientIndex: number }
> = new Map();
for (let i = 0; i < REDIS_CLIENTS.length; i++) {
const prefix = RedisClientPrefix[REDIS_CLIENTS[i]];
redisClients.push(new RedisClient({ prefix }));
await redisClients[i].connect();
{
client: RedisClient;
clientIndex: number;
lastRotationTime: number;
lock: boolean;
}
> = new Map();
if (useRedis) {
logger.info('Connecting to redis');
for (let i = 0; i < REDIS_CLIENTS.length; i++) {
redisClients.push(new RedisClient({ prefix: REDIS_CLIENTS[i] }));
}
for (let i = 0; i < sdkConfig.SPOT_MARKETS.length; i++) {
spotMarketRedisMap.set(sdkConfig.SPOT_MARKETS[i].marketIndex, {
client: redisClients[0],
clientIndex: 0,
lastRotationTime: 0,
lock: false,
});
}
for (let i = 0; i < sdkConfig.PERP_MARKETS.length; i++) {
perpMarketRedisMap.set(sdkConfig.PERP_MARKETS[i].marketIndex, {
client: redisClients[0],
clientIndex: 0,
lastRotationTime: 0,
lock: false,
});
}
}
// Slot subscriber for source
const connection = new Connection(endpoint, {
commitment: stateCommitment,
wsEndpoint,
});
const slotSubscriber = new SlotSubscriber(connection);
await slotSubscriber.subscribe();
function canRotate(marketType: MarketType, marketIndex: number) {
if (isVariant(marketType, 'spot')) {
const state = spotMarketRedisMap.get(marketIndex);
if (state) {
const now = Date.now();
if (now - state.lastRotationTime > ROTATION_COOLDOWN && !state.lock) {
state.lastRotationTime = now;
return true;
}
}
} else {
const state = perpMarketRedisMap.get(marketIndex);
if (state) {
const now = Date.now();
if (now - state.lastRotationTime > ROTATION_COOLDOWN && !state.lock) {
state.lastRotationTime = now;
return true;
}
}
}
return false;
}
app.get(
'/health',
handleHealthCheck(2 * SLOT_STALENESS_TOLERANCE * 400, slotSubscriber)
function rotateClient(marketType: MarketType, marketIndex: number) {
if (isVariant(marketType, 'spot')) {
const state = spotMarketRedisMap.get(marketIndex);
if (state) {
state.lock = true;
const nextClientIndex = (state.clientIndex + 1) % redisClients.length;
state.client = redisClients[nextClientIndex];
state.clientIndex = nextClientIndex;
logger.info(
`Rotated redis client to index ${nextClientIndex} for spot market ${marketIndex}`
);
app.get(
'/',
handleHealthCheck(2 * SLOT_STALENESS_TOLERANCE * 400, slotSubscriber)
state.lastRotationTime = Date.now();
state.lock = false;
}
} else {
const state = perpMarketRedisMap.get(marketIndex);
if (state) {
state.lock = true;
const nextClientIndex = (state.clientIndex + 1) % redisClients.length;
state.client = redisClients[nextClientIndex];
state.clientIndex = nextClientIndex;
logger.info(
`Rotated redis client to index ${nextClientIndex} for perp market ${marketIndex}`
);
state.lastRotationTime = Date.now();
state.lock = false;
}
}
}
const handleStartup = async (_req, res, _next) => {
let healthy = false;
for (const redisClient of redisClients) {
if (redisClient.connected) {
healthy = true;
}
}
if (healthy) {
if (slotSubscriber.currentSlot && !redisClients.some((c) => !c.connected)) {
res.writeHead(200);
res.end('OK');
} else {
@@ -171,7 +229,16 @@ const main = async () => {
res.end('Not ready');
}
};
app.get(
'/health',
handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, slotSubscriber)
);
app.get('/startup', handleStartup);
app.get(
'/',
handleHealthCheck(2 * WS_FALLBACK_FETCH_INTERVAL, slotSubscriber)
);
app.get('/l2', async (req, res, next) => {
try {
@@ -182,111 +249,58 @@ const main = async () => {
includeVamm,
includePhoenix,
includeSerum,
grouping, // undefined or PRICE_PRECISION
includeOracle,
} = req.query;
const { normedMarketType, normedMarketIndex, error } = validateDlobQuery(
driftEnv,
marketType as string,
marketIndex as string
);
if (error) {
res.status(400).send(error);
return;
}
const isSpot = isVariant(normedMarketType, 'spot');
let adjustedDepth = depth ?? '10';
if (grouping !== undefined) {
// If grouping is also supplied, we want the entire book depth.
// we will apply depth after grouping
adjustedDepth = '-1';
}
const isSpot = (marketType as string).toLowerCase() === 'spot';
const normedMarketIndex = parseInt(marketIndex as string);
const normedMarketType = isSpot ? MarketType.SPOT : MarketType.PERP;
const adjustedDepth = depth ?? '100';
let l2Formatted: any;
let slotDiff: number;
if (
!isSpot &&
`${includeVamm}`?.toLowerCase() === 'true' &&
`${includeOracle}`?.toLowerCase().toLowerCase() === 'true' &&
!grouping
) {
let redisL2: string;
if (!isSpot && `${includeVamm}`?.toLowerCase() === 'true') {
const redisClient = perpMarketRedisMap.get(normedMarketIndex).client;
if (parseInt(adjustedDepth as string) === 5) {
redisL2 = await redisClient.getRaw(
`last_update_orderbook_perp_${normedMarketIndex}_depth_5`
const redisL2 = await redisClient.get(
`last_update_orderbook_perp_${normedMarketIndex}`
);
} else if (parseInt(adjustedDepth as string) === 20) {
redisL2 = await redisClient.getRaw(
`last_update_orderbook_perp_${normedMarketIndex}_depth_20`
);
} else if (parseInt(adjustedDepth as string) === 100) {
redisL2 = await redisClient.getRaw(
`last_update_orderbook_perp_${normedMarketIndex}_depth_100`
);
}
if (redisL2) {
slotDiff =
slotSubscriber.getSlot() - parseInt(JSON.parse(redisL2).slot);
if (slotDiff < SLOT_STALENESS_TOLERANCE) {
l2Formatted = redisL2;
const depth = Math.min(parseInt(adjustedDepth as string) ?? 1, 100);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
if (
redisL2 &&
slotSubscriber.getSlot() - parseInt(redisL2['slot']) <
SLOT_STALENESS_TOLERANCE
) {
l2Formatted = JSON.stringify(redisL2);
} else {
if (redisClients.length > 1) {
const nextClientIndex =
(perpMarketRedisMap.get(normedMarketIndex).clientIndex + 1) %
redisClients.length;
perpMarketRedisMap.set(normedMarketIndex, {
client: redisClients[nextClientIndex],
clientIndex: nextClientIndex,
});
console.log(
`Rotated redis client to index ${nextClientIndex} for perp market ${normedMarketIndex}`
);
if (redisL2 && redisClients.length > 1) {
if (canRotate(normedMarketType, normedMarketIndex)) {
rotateClient(normedMarketType, normedMarketIndex);
}
}
}
} else if (
isSpot &&
`${includeSerum}`?.toLowerCase() === 'true' &&
`${includePhoenix}`?.toLowerCase() === 'true' &&
`${includeOracle}`?.toLowerCase() === 'true' &&
!grouping
`${includePhoenix}`?.toLowerCase() === 'true'
) {
let redisL2: string;
const redisClient = spotMarketRedisMap.get(normedMarketIndex).client;
if (parseInt(adjustedDepth as string) === 5) {
redisL2 = await redisClient.getRaw(
`last_update_orderbook_spot_${normedMarketIndex}_depth_5`
const redisL2 = await redisClient.get(
`last_update_orderbook_spot_${normedMarketIndex}`
);
} else if (parseInt(adjustedDepth as string) === 20) {
redisL2 = await redisClient.getRaw(
`last_update_orderbook_spot_${normedMarketIndex}_depth_20`
);
} else if (parseInt(adjustedDepth as string) === 100) {
redisL2 = await redisClient.getRaw(
`last_update_orderbook_spot_${normedMarketIndex}_depth_100`
);
}
if (redisL2) {
slotDiff =
slotSubscriber.getSlot() - parseInt(JSON.parse(redisL2).slot);
if (slotDiff < SLOT_STALENESS_TOLERANCE) {
l2Formatted = redisL2;
const depth = Math.min(parseInt(adjustedDepth as string) ?? 1, 100);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
if (
redisL2 &&
slotSubscriber.getSlot() - parseInt(redisL2['slot']) <
SLOT_STALENESS_TOLERANCE
) {
l2Formatted = JSON.stringify(redisL2);
} else {
if (redisClients.length > 1) {
const nextClientIndex =
(spotMarketRedisMap.get(normedMarketIndex).clientIndex + 1) %
redisClients.length;
spotMarketRedisMap.set(normedMarketIndex, {
client: redisClients[nextClientIndex],
clientIndex: nextClientIndex,
});
console.log(
`Rotated redis client to index ${nextClientIndex} for spot market ${normedMarketIndex}`
);
if (redisL2 && redisClients.length > 1) {
if (canRotate(normedMarketType, normedMarketIndex)) {
rotateClient(normedMarketType, normedMarketIndex);
}
}
}
@@ -295,185 +309,15 @@ const main = async () => {
if (l2Formatted) {
cacheHitCounter.add(1, {
miss: false,
path: req.baseUrl + req.path,
});
res.writeHead(200);
res.end(l2Formatted);
return;
} else {
if (slotDiff) {
res.writeHead(500);
res.end(`Slot too stale : ${slotDiff}`);
} else {
res.writeHead(400);
res.end(`Bad Request: no cached L2 found`);
res.end({ error: 'No cached L2 found' });
}
}
} catch (err) {
next(err);
}
});
app.get('/batchL2', async (req, res, next) => {
try {
const {
marketName,
marketIndex,
marketType,
depth,
includeVamm,
includePhoenix,
includeSerum,
includeOracle,
grouping, // undefined or PRICE_PRECISION
} = req.query;
const normedParams = normalizeBatchQueryParams({
marketName: marketName as string | undefined,
marketIndex: marketIndex as string | undefined,
marketType: marketType as string | undefined,
depth: depth as string | undefined,
includeVamm: includeVamm as string | undefined,
includePhoenix: includePhoenix as string | undefined,
includeSerum: includeSerum as string | undefined,
includeOracle: includeOracle as string | undefined,
grouping: grouping as string | undefined,
});
if (normedParams === undefined) {
res
.status(400)
.send(
'Bad Request: all params for batch request must be the same length'
);
return;
}
const l2s = await Promise.all(
normedParams.map(async (normedParam) => {
const { normedMarketType, normedMarketIndex, error } =
validateDlobQuery(
driftEnv,
normedParam['marketType'] as string,
normedParam['marketIndex'] as string
);
if (error) {
res.status(400).send(`Bad Request: ${error}`);
return;
}
const isSpot = isVariant(normedMarketType, 'spot');
let adjustedDepth = normedParam['depth'] ?? '10';
if (normedParam['grouping'] !== undefined) {
// If grouping is also supplied, we want the entire book depth.
// we will apply depth after grouping
adjustedDepth = '-1';
}
let l2Formatted: any;
if (
!isSpot &&
normedParam['includeVamm']?.toLowerCase() === 'true' &&
normedParam['includeOracle']?.toLowerCase() === 'true' &&
!normedParam['grouping']
) {
let redisL2: string;
const redisClient =
perpMarketRedisMap.get(normedMarketIndex).client;
if (parseInt(adjustedDepth as string) === 5) {
redisL2 = await redisClient.getRaw(
`last_update_orderbook_perp_${normedMarketIndex}_depth_5`
);
} else if (parseInt(adjustedDepth as string) === 20) {
redisL2 = await redisClient.getRaw(
`last_update_orderbook_perp_${normedMarketIndex}_depth_20`
);
} else if (parseInt(adjustedDepth as string) === 100) {
redisL2 = await redisClient.getRaw(
`last_update_orderbook_perp_${normedMarketIndex}_depth_100`
);
}
if (redisL2) {
const parsedRedisL2 = JSON.parse(redisL2);
if (
slotSubscriber.getSlot() - parseInt(parsedRedisL2.slot) <
SLOT_STALENESS_TOLERANCE
) {
l2Formatted = parsedRedisL2;
} else {
if (redisClients.length > 1) {
const nextClientIndex =
(perpMarketRedisMap.get(normedMarketIndex).clientIndex +
1) %
redisClients.length;
perpMarketRedisMap.set(normedMarketIndex, {
client: redisClients[nextClientIndex],
clientIndex: nextClientIndex,
});
console.log(
`Rotated redis client to index ${nextClientIndex} for perp market ${normedMarketIndex}`
);
}
}
}
} else if (
isSpot &&
normedParam['includePhoenix']?.toLowerCase() === 'true' &&
normedParam['includeSerum']?.toLowerCase() === 'true' &&
!normedParam['grouping']
) {
let redisL2: string;
const redisClient =
spotMarketRedisMap.get(normedMarketIndex).client;
if (parseInt(adjustedDepth as string) === 5) {
redisL2 = await redisClient.getRaw(
`last_update_orderbook_spot_${normedMarketIndex}_depth_5`
);
} else if (parseInt(adjustedDepth as string) === 20) {
redisL2 = await redisClient.getRaw(
`last_update_orderbook_spot_${normedMarketIndex}_depth_20`
);
} else if (parseInt(adjustedDepth as string) === 100) {
redisL2 = await redisClient.getRaw(
`last_update_orderbook_spot_${normedMarketIndex}_depth_100`
);
}
if (redisL2) {
const parsedRedisL2 = JSON.parse(redisL2);
if (
slotSubscriber.getSlot() - parseInt(parsedRedisL2.slot) <
SLOT_STALENESS_TOLERANCE
) {
l2Formatted = parsedRedisL2;
} else {
if (redisClients.length > 1) {
const nextClientIndex =
(spotMarketRedisMap.get(normedMarketIndex).clientIndex +
1) %
redisClients.length;
spotMarketRedisMap.set(normedMarketIndex, {
client: redisClients[nextClientIndex],
clientIndex: nextClientIndex,
});
console.log(
`Rotated redis client to index ${nextClientIndex} for spot market ${normedMarketIndex}`
);
}
}
}
if (l2Formatted) {
cacheHitCounter.add(1, {
miss: false,
});
return l2Formatted;
}
}
})
);
res.writeHead(200);
res.end(JSON.stringify({ l2s }));
} catch (err) {
next(err);
}
@@ -484,65 +328,7 @@ const main = async () => {
});
};
export const validateDlobQuery = (
driftEnv: DriftEnv,
marketType?: string,
marketIndex?: string
): {
normedMarketType?: MarketType;
normedMarketIndex?: number;
error?: string;
} => {
let normedMarketType: MarketType = undefined;
let normedMarketIndex: number = undefined;
if (marketIndex === undefined || marketType === undefined) {
return {
error:
'Bad Request: (marketName) or (marketIndex and marketType) must be supplied',
};
}
// validate marketType
switch ((marketType as string).toLowerCase()) {
case 'spot': {
normedMarketType = MarketType.SPOT;
normedMarketIndex = parseInt(marketIndex as string);
const spotMarketIndicies = SpotMarkets[driftEnv].map(
(mkt) => mkt.marketIndex
);
if (!spotMarketIndicies.includes(normedMarketIndex)) {
return {
error: 'Bad Request: invalid marketIndex',
};
}
break;
}
case 'perp': {
normedMarketType = MarketType.PERP;
normedMarketIndex = parseInt(marketIndex as string);
const perpMarketIndicies = PerpMarkets[driftEnv].map(
(mkt) => mkt.marketIndex
);
if (!perpMarketIndicies.includes(normedMarketIndex)) {
return {
error: 'Bad Request: invalid marketIndex',
};
}
break;
}
default:
return {
error: 'Bad Request: marketType must be either "spot" or "perp"',
};
}
return {
normedMarketType,
normedMarketIndex,
};
};
async function recursiveTryCatch(f: () => void) {
async function recursiveTryCatch(f: () => Promise<void>) {
try {
await f();
} catch (e) {
@@ -554,4 +340,4 @@ async function recursiveTryCatch(f: () => void) {
recursiveTryCatch(() => main());
export { commitHash, endpoint, sdkConfig, wsEndpoint };
export { commitHash, driftEnv, endpoint, sdkConfig, wsEndpoint };