trade/trade-drift-dlob
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore: update to common redis + top markers account flag

Browse Source
This commit is contained in:
Jack Waller
2024-05-16 12:39:44 +10:00
parent cd26f39c72
commit 956a031fec
13 changed files with 1699 additions and 316 deletions
Download Patch File Download Diff File Expand all files Collapse all files

123
src/index.ts
View File

@@ -19,6 +19,7 @@ import {
OrderSubscriber,
MarketType,
} from '@drift-labs/sdk';
import { RedisClient, RedisClientPrefix } from '@drift/common';
import { logger, setLogLevel } from './utils/logger';
@@ -42,43 +43,19 @@ import {
normalizeBatchQueryParams,
sleep,
validateDlobQuery,
getAccountFromId,
} from './utils/utils';
import FEATURE_FLAGS from './utils/featureFlags';
import { getDLOBProviderFromOrderSubscriber } from './dlobProvider';
import { RedisClient } from './utils/redisClient';
require('dotenv').config();
// Reading in Redis env vars
const REDIS_HOSTS = process.env.REDIS_HOSTS?.replace(/^\[|\]$/g, '')
const REDIS_CLIENTS = process.env.REDIS_CLIENTS?.replace(/^\[|\]$/g, '')
.split(',')
.map((host) => host.trim()) || ['localhost'];
const REDIS_PORTS = process.env.REDIS_PORTS?.replace(/^\[|\]$/g, '')
.split(',')
.map((port) => parseInt(port.trim(), 10)) || [6379];
const REDIS_PASSWORDS_ENV = process.env.REDIS_PASSWORDS || "['']";
.map((clients) => clients.trim()) || ['DLOB'];
let REDIS_PASSWORDS;
if (REDIS_PASSWORDS_ENV.trim() === "['']") {
REDIS_PASSWORDS = [undefined];
} else {
REDIS_PASSWORDS = REDIS_PASSWORDS_ENV.replace(/^\[|\]$/g, '')
.split(/\s*,\s*/)
.map((pwd) => pwd.replace(/(^'|'$)/g, '').trim())
.map((pwd) => (pwd === '' ? undefined : pwd));
}
console.log('Redis Hosts:', REDIS_HOSTS);
console.log('Redis Ports:', REDIS_PORTS);
console.log('Redis Passwords:', REDIS_PASSWORDS);
if (
REDIS_PORTS.length !== REDIS_PASSWORDS.length ||
REDIS_PORTS.length !== REDIS_HOSTS.length
) {
throw 'REDIS_HOSTS and REDIS_PASSWORDS and REDIS_PORTS must be the same length';
}
console.log('Redis Clients:', REDIS_CLIENTS);
const driftEnv = (process.env.ENV || 'devnet') as DriftEnv;
const commitHash = process.env.COMMIT;
@@ -316,16 +293,11 @@ const main = async (): Promise<void> => {
> = new Map();
if (useRedis) {
logger.info('Connecting to redis');
for (let i = 0; i < REDIS_HOSTS.length; i++) {
redisClients.push(
new RedisClient(
REDIS_HOSTS[i],
REDIS_PORTS[i].toString(),
REDIS_PASSWORDS[i] || undefined
)
);
await redisClients[i].connect();
for (let i = 0; i < REDIS_CLIENTS.length; i++) {
const prefix = RedisClientPrefix[REDIS_CLIENTS[i]];
redisClients.push(new RedisClient({ prefix }));
}
for (let i = 0; i < sdkConfig.SPOT_MARKETS.length; i++) {
spotMarketRedisMap.set(sdkConfig.SPOT_MARKETS[i].marketIndex, {
client: redisClients[0],
@@ -344,6 +316,8 @@ const main = async (): Promise<void> => {
}
}
const userMapClient = new RedisClient({ prefix: RedisClientPrefix.USER_MAP });
function canRotate(marketType: MarketType, marketIndex: number) {
if (isVariant(marketType, 'spot')) {
const state = spotMarketRedisMap.get(marketIndex);
@@ -427,15 +401,21 @@ const main = async (): Promise<void> => {
try {
const { marketIndex, marketType } = req.query;
const fees = await redisClients[
parseInt(process.env.HELIUS_REDIS_HOST_INDEX) ?? 0
].client.get(`priorityFees_${marketType}_${marketIndex}`);
const fees = await redisClients
.find(
(client) =>
client.forceGetClient().options.keyPrefix ===
RedisClientPrefix.DLOB_HELIUS
)
.getRaw(`priorityFees_${marketType}_${marketIndex}`);
if (fees) {
res.status(200).json({
...JSON.parse(fees),
marketType,
marketIndex,
});
return;
} else {
res.writeHead(404);
@@ -466,11 +446,16 @@ const main = async (): Promise<void> => {
const fees = await Promise.all(
normedParams.map(async (normedParam) => {
const fees = await redisClients[
parseInt(process.env.HELIUS_REDIS_HOST_INDEX) ?? 0
].client.get(
`priorityFees_${normedParam['marketType']}_${normedParam['marketIndex']}`
);
const fees = await redisClients
.find(
(client) =>
client.forceGetClient().options.keyPrefix ===
RedisClientPrefix.DLOB_HELIUS
)
.getRaw(
`priorityFees_${normedParam['marketType']}_${normedParam['marketIndex']}`
);
return {
...JSON.parse(fees),
marketType: normedParam['marketType'],
@@ -499,6 +484,7 @@ const main = async (): Promise<void> => {
marketType,
side, // bid or ask
limit,
includeAccounts,
} = req.query;
const { normedMarketType, normedMarketIndex, error } = validateDlobQuery(
@@ -533,12 +519,17 @@ const main = async (): Promise<void> => {
normedLimit = 4;
}
let accountFlag = false;
if (includeAccounts) {
accountFlag = includeAccounts === 'true';
}
let topMakers: string[];
if (useRedis) {
const redisClient = isVariant(normedMarketType, 'perp')
? perpMarketRedisMap.get(normedMarketIndex).client
: spotMarketRedisMap.get(normedMarketIndex).client;
const redisResponse = await redisClient.client.get(
const redisResponse = await redisClient.getRaw(
`last_update_orderbook_best_makers_${getVariant(
normedMarketType
)}_${marketIndex}`
@@ -565,6 +556,13 @@ const main = async (): Promise<void> => {
path: req.baseUrl + req.path,
});
res.writeHead(200);
if (accountFlag) {
const topAccounts = await getAccountFromId(userMapClient, topMakers);
res.end(JSON.stringify(topAccounts));
return;
}
res.end(JSON.stringify(topMakers));
return;
}
@@ -619,6 +617,13 @@ const main = async (): Promise<void> => {
path: req.baseUrl + req.path,
});
res.writeHead(200);
if (accountFlag) {
const topAccounts = await getAccountFromId(userMapClient, topMakers);
res.end(JSON.stringify(topAccounts));
return;
}
res.end(JSON.stringify(topMakers));
} catch (err) {
next(err);
@@ -661,15 +666,15 @@ const main = async (): Promise<void> => {
let redisL2: string;
const redisClient = perpMarketRedisMap.get(normedMarketIndex).client;
if (parseInt(adjustedDepth as string) === 5) {
redisL2 = await redisClient.client.get(
redisL2 = await redisClient.getRaw(
`last_update_orderbook_perp_${normedMarketIndex}_depth_5`
);
} else if (parseInt(adjustedDepth as string) === 20) {
redisL2 = await redisClient.client.get(
redisL2 = await redisClient.getRaw(
`last_update_orderbook_perp_${normedMarketIndex}_depth_20`
);
} else if (parseInt(adjustedDepth as string) === 100) {
redisL2 = await redisClient.client.get(
redisL2 = await redisClient.getRaw(
`last_update_orderbook_perp_${normedMarketIndex}_depth_100`
);
}
@@ -694,15 +699,15 @@ const main = async (): Promise<void> => {
let redisL2: string;
const redisClient = spotMarketRedisMap.get(normedMarketIndex).client;
if (parseInt(adjustedDepth as string) === 5) {
redisL2 = await redisClient.client.get(
redisL2 = await redisClient.getRaw(
`last_update_orderbook_spot_${normedMarketIndex}_depth_5`
);
} else if (parseInt(adjustedDepth as string) === 20) {
redisL2 = await redisClient.client.get(
redisL2 = await redisClient.getRaw(
`last_update_orderbook_spot_${normedMarketIndex}_depth_20`
);
} else if (parseInt(adjustedDepth as string) === 100) {
redisL2 = await redisClient.client.get(
redisL2 = await redisClient.getRaw(
`last_update_orderbook_spot_${normedMarketIndex}_depth_100`
);
}
@@ -834,15 +839,15 @@ const main = async (): Promise<void> => {
const redisClient =
perpMarketRedisMap.get(normedMarketIndex).client;
if (parseInt(adjustedDepth as string) === 5) {
redisL2 = await redisClient.client.get(
redisL2 = await redisClient.getRaw(
`last_update_orderbook_perp_${normedMarketIndex}_depth_5`
);
} else if (parseInt(adjustedDepth as string) === 20) {
redisL2 = await redisClient.client.get(
redisL2 = await redisClient.getRaw(
`last_update_orderbook_perp_${normedMarketIndex}_depth_20`
);
} else if (parseInt(adjustedDepth as string) === 100) {
redisL2 = await redisClient.client.get(
redisL2 = await redisClient.getRaw(
`last_update_orderbook_perp_${normedMarketIndex}_depth_100`
);
}
@@ -870,15 +875,15 @@ const main = async (): Promise<void> => {
const redisClient =
spotMarketRedisMap.get(normedMarketIndex).client;
if (parseInt(adjustedDepth as string) === 5) {
redisL2 = await redisClient.client.get(
redisL2 = await redisClient.getRaw(
`last_update_orderbook_spot_${normedMarketIndex}_depth_5`
);
} else if (parseInt(adjustedDepth as string) === 20) {
redisL2 = await redisClient.client.get(
redisL2 = await redisClient.getRaw(
`last_update_orderbook_spot_${normedMarketIndex}_depth_20`
);
} else if (parseInt(adjustedDepth as string) === 100) {
redisL2 = await redisClient.client.get(
redisL2 = await redisClient.getRaw(
`last_update_orderbook_spot_${normedMarketIndex}_depth_100`
);
}
@@ -976,7 +981,7 @@ const main = async (): Promise<void> => {
const redisClient = (
marketTypeStr === 'spot' ? spotMarketRedisMap : perpMarketRedisMap
).get(normedMarketIndex).client;
const redisL3 = await redisClient.client.get(
const redisL3 = await redisClient.getRaw(
`last_update_orderbook_l3_${marketTypeStr}_${normedMarketIndex}`
);
if (