Nour/improved rotation (#290)

* improved rotation

* fix bugs

* only parse slot if necessary

* add metrics for ws source

* add select by recent slot to utils

* fix solana web3.js version

---------

Co-authored-by: wphan <william@drift.trade>
This commit is contained in:
moosecat
2024-12-03 13:47:02 -08:00
committed by GitHub
parent d539622b13
commit 2a6899384c
10 changed files with 307 additions and 383 deletions

View File

@@ -8,6 +8,7 @@ import {
View,
} from '@opentelemetry/sdk-metrics-base';
import { SlotSource } from '@drift-labs/sdk';
require('dotenv').config();
/**
* Creates {count} buckets of size {increment} starting from {start}. Each bucket stores the count of values within its "size".

View File

@@ -17,7 +17,6 @@ import {
initialize,
isVariant,
OrderSubscriber,
MarketType,
PhoenixSubscriber,
BulkAccountLoader,
isOperationPaused,
@@ -49,6 +48,7 @@ import {
getAccountFromId,
getRawAccountFromId,
getOpenbookSubscriber,
selectMostRecentBySlot,
} from './utils/utils';
import FEATURE_FLAGS from './utils/featureFlags';
import { getDLOBProviderFromOrderSubscriber } from './dlobProvider';
@@ -84,10 +84,8 @@ const serverPort = process.env.PORT || 6969;
export const ORDERBOOK_UPDATE_INTERVAL = 1000;
const WS_FALLBACK_FETCH_INTERVAL = ORDERBOOK_UPDATE_INTERVAL * 60;
const SLOT_STALENESS_TOLERANCE =
parseInt(process.env.SLOT_STALENESS_TOLERANCE) || 35;
const ROTATION_COOLDOWN = parseInt(process.env.ROTATION_COOLDOWN) || 5000;
parseInt(process.env.SLOT_STALENESS_TOLERANCE) || 1000;
const useWebsocket = process.env.USE_WEBSOCKET?.toLowerCase() === 'true';
const useRedis = process.env.USE_REDIS?.toLowerCase() === 'true';
const logFormat =
':remote-addr - :remote-user [:date[clf]] ":method :url HTTP/:http-version" :status :res[content-length] ":referrer" ":user-agent" :req[x-forwarded-for]';
@@ -299,107 +297,27 @@ const main = async (): Promise<void> => {
// Handle redis client initialization and rotation maps
const redisClients: Array<RedisClient> = [];
const spotMarketRedisMap: Map<
number,
{
client: RedisClient;
clientIndex: number;
lastRotationTime: number;
lock: boolean;
}
> = new Map();
const perpMarketRedisMap: Map<
number,
{
client: RedisClient;
clientIndex: number;
lastRotationTime: number;
lock: boolean;
}
> = new Map();
if (useRedis) {
logger.info('Connecting to redis');
for (let i = 0; i < REDIS_CLIENTS.length; i++) {
redisClients.push(new RedisClient({ prefix: REDIS_CLIENTS[i] }));
}
for (let i = 0; i < sdkConfig.SPOT_MARKETS.length; i++) {
spotMarketRedisMap.set(sdkConfig.SPOT_MARKETS[i].marketIndex, {
client: redisClients[0],
clientIndex: 0,
lastRotationTime: 0,
lock: false,
});
}
for (let i = 0; i < sdkConfig.PERP_MARKETS.length; i++) {
perpMarketRedisMap.set(sdkConfig.PERP_MARKETS[i].marketIndex, {
client: redisClients[0],
clientIndex: 0,
lastRotationTime: 0,
lock: false,
});
}
logger.info('Connecting to redis');
for (let i = 0; i < REDIS_CLIENTS.length; i++) {
redisClients.push(new RedisClient({ prefix: REDIS_CLIENTS[i] }));
}
const fetchFromRedis = async (
key: string,
selectionCriteria: (responses: any) => any
): Promise<JSON> => {
const redisResponses = await Promise.all(
redisClients.map((client) => client.getRaw(key))
);
return selectionCriteria(redisResponses);
};
const userMapClient = new RedisClient({
host: process.env.ELASTICACHE_USERMAP_HOST ?? process.env.ELASTICACHE_HOST,
port: process.env.ELASTICACHE_USERMAP_PORT ?? process.env.ELASTICACHE_PORT,
prefix: RedisClientPrefix.USER_MAP,
});
function canRotate(marketType: MarketType, marketIndex: number) {
if (isVariant(marketType, 'spot')) {
const state = spotMarketRedisMap.get(marketIndex);
if (state) {
const now = Date.now();
if (now - state.lastRotationTime > ROTATION_COOLDOWN && !state.lock) {
state.lastRotationTime = now;
return true;
}
}
} else {
const state = perpMarketRedisMap.get(marketIndex);
if (state) {
const now = Date.now();
if (now - state.lastRotationTime > ROTATION_COOLDOWN && !state.lock) {
state.lastRotationTime = now;
return true;
}
}
}
return false;
}
function rotateClient(marketType: MarketType, marketIndex: number) {
if (isVariant(marketType, 'spot')) {
const state = spotMarketRedisMap.get(marketIndex);
if (state) {
state.lock = true;
const nextClientIndex = (state.clientIndex + 1) % redisClients.length;
state.client = redisClients[nextClientIndex];
state.clientIndex = nextClientIndex;
logger.info(
`Rotated redis client to index ${nextClientIndex} for spot market ${marketIndex}`
);
state.lastRotationTime = Date.now();
state.lock = false;
}
} else {
const state = perpMarketRedisMap.get(marketIndex);
if (state) {
state.lock = true;
const nextClientIndex = (state.clientIndex + 1) % redisClients.length;
state.client = redisClients[nextClientIndex];
state.clientIndex = nextClientIndex;
logger.info(
`Rotated redis client to index ${nextClientIndex} for perp market ${marketIndex}`
);
state.lastRotationTime = Date.now();
state.lock = false;
}
}
}
logger.info(`Initializing all market subscribers...`);
const initAllMarketSubscribersStart = Date.now();
MARKET_SUBSCRIBERS = await initializeAllMarketSubscribers(driftClient);
@@ -555,27 +473,22 @@ const main = async (): Promise<void> => {
}
let topMakers: string[];
if (useRedis) {
const redisClient = isVariant(normedMarketType, 'perp')
? perpMarketRedisMap.get(normedMarketIndex).client
: spotMarketRedisMap.get(normedMarketIndex).client;
const redisResponse = await redisClient.getRaw(
`last_update_orderbook_best_makers_${getVariant(
normedMarketType
)}_${marketIndex}`
);
if (redisResponse) {
const parsedResponse = JSON.parse(redisResponse);
if (
parsedResponse &&
dlobProvider.getSlot() - parsedResponse.slot <
SLOT_STALENESS_TOLERANCE
) {
if (side === 'bid') {
topMakers = parsedResponse.bids;
} else {
topMakers = parsedResponse.asks;
}
const redisResponse = await fetchFromRedis(
`last_update_orderbook_best_makers_${getVariant(
normedMarketType
)}_${marketIndex}`,
selectMostRecentBySlot
);
if (redisResponse) {
if (
dlobProvider.getSlot() - redisResponse['slot'] <
SLOT_STALENESS_TOLERANCE
) {
if (side === 'bid') {
topMakers = redisResponse['bids'];
} else {
topMakers = redisResponse['asks'];
}
}
}
@@ -729,54 +642,38 @@ const main = async (): Promise<void> => {
const adjustedDepth = depth ?? '100';
let l2Formatted: any;
if (useRedis) {
if (!isSpot && `${includeVamm}`?.toLowerCase() === 'true') {
const redisClient = perpMarketRedisMap.get(normedMarketIndex).client;
const redisL2 = await redisClient.get(
`last_update_orderbook_perp_${normedMarketIndex}`
);
const depth = Math.min(parseInt(adjustedDepth as string) ?? 1, 100);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
if (!isSpot && `${includeVamm}`?.toLowerCase() === 'true') {
const redisL2 = await fetchFromRedis(
`last_update_orderbook_perp_${normedMarketIndex}`,
selectMostRecentBySlot
);
const depth = Math.min(parseInt(adjustedDepth as string) ?? 1, 100);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
if (
redisL2 &&
dlobProvider.getSlot() - parseInt(redisL2['slot']) <
SLOT_STALENESS_TOLERANCE
) {
l2Formatted = JSON.stringify(redisL2);
} else {
if (redisL2 && redisClients.length > 1) {
if (canRotate(normedMarketType, normedMarketIndex)) {
rotateClient(normedMarketType, normedMarketIndex);
}
}
}
} else if (
isSpot &&
`${includePhoenix}`?.toLowerCase() === 'true' &&
`${includeOpenbook}`?.toLowerCase() === 'true'
if (
redisL2 &&
dlobProvider.getSlot() - redisL2['slot'] < SLOT_STALENESS_TOLERANCE
) {
const redisClient = spotMarketRedisMap.get(normedMarketIndex).client;
const redisL2 = await redisClient.get(
`last_update_orderbook_spot_${normedMarketIndex}`
);
const depth = Math.min(parseInt(adjustedDepth as string) ?? 1, 100);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
if (
redisL2 &&
dlobProvider.getSlot() - parseInt(redisL2['slot']) <
SLOT_STALENESS_TOLERANCE
) {
l2Formatted = JSON.stringify(redisL2);
} else {
if (redisL2 && redisClients.length > 1) {
if (canRotate(normedMarketType, normedMarketIndex)) {
rotateClient(normedMarketType, normedMarketIndex);
}
}
}
l2Formatted = JSON.stringify(redisL2);
}
} else if (
isSpot &&
`${includePhoenix}`?.toLowerCase() === 'true' &&
`${includeOpenbook}`?.toLowerCase() === 'true'
) {
const redisL2 = await fetchFromRedis(
`last_update_orderbook_spot_${normedMarketIndex}`,
selectMostRecentBySlot
);
const depth = Math.min(parseInt(adjustedDepth as string) ?? 1, 100);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
if (
redisL2 &&
dlobProvider.getSlot() - redisL2['slot'] < SLOT_STALENESS_TOLERANCE
) {
l2Formatted = JSON.stringify(redisL2);
}
if (l2Formatted) {
@@ -891,65 +788,40 @@ const main = async (): Promise<void> => {
const adjustedDepth = normedParam['depth'] ?? '100';
let l2Formatted: any;
if (useRedis) {
if (
!isSpot &&
normedParam['includeVamm']?.toLowerCase() === 'true'
) {
const redisClient =
perpMarketRedisMap.get(normedMarketIndex).client;
const redisL2 = await redisClient.get(
`last_update_orderbook_perp_${normedMarketIndex}`
);
const depth = Math.min(
parseInt(adjustedDepth as string) ?? 1,
100
);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
if (redisL2) {
if (
dlobProvider.getSlot() - parseInt(redisL2['slot']) <
SLOT_STALENESS_TOLERANCE
) {
l2Formatted = redisL2;
} else {
if (redisClients.length > 1) {
if (canRotate(normedMarketType, normedMarketIndex)) {
rotateClient(normedMarketType, normedMarketIndex);
}
}
}
if (!isSpot && normedParam['includeVamm']?.toLowerCase() === 'true') {
const redisL2 = await fetchFromRedis(
`last_update_orderbook_perp_${normedMarketIndex}`,
selectMostRecentBySlot
);
const depth = Math.min(parseInt(adjustedDepth as string) ?? 1, 100);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
if (redisL2) {
if (
dlobProvider.getSlot() - redisL2['slot'] <
SLOT_STALENESS_TOLERANCE
) {
l2Formatted = redisL2;
}
} else if (
isSpot &&
normedParam['includePhoenix']?.toLowerCase() === 'true' &&
normedParam['includeOpenbook']?.toLowerCase() === 'true'
) {
const redisClient =
spotMarketRedisMap.get(normedMarketIndex).client;
const redisL2 = await redisClient.get(
`last_update_orderbook_spot_${normedMarketIndex}`
);
const depth = Math.min(
parseInt(adjustedDepth as string) ?? 1,
100
);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
if (redisL2) {
if (
dlobProvider.getSlot() - parseInt(redisL2['slot']) <
SLOT_STALENESS_TOLERANCE
) {
l2Formatted = redisL2;
} else {
if (redisClients.length > 1) {
if (canRotate(normedMarketType, normedMarketIndex)) {
rotateClient(normedMarketType, normedMarketIndex);
}
}
}
}
} else if (
isSpot &&
normedParam['includePhoenix']?.toLowerCase() === 'true' &&
normedParam['includeOpenbook']?.toLowerCase() === 'true'
) {
const redisL2 = await fetchFromRedis(
`last_update_orderbook_spot_${normedMarketIndex}`,
selectMostRecentBySlot
);
const depth = Math.min(parseInt(adjustedDepth as string) ?? 1, 100);
redisL2['bids'] = redisL2['bids']?.slice(0, depth);
redisL2['asks'] = redisL2['asks']?.slice(0, depth);
if (redisL2) {
if (
dlobProvider.getSlot() - redisL2['slot'] <
SLOT_STALENESS_TOLERANCE
) {
l2Formatted = redisL2;
}
}
@@ -1033,31 +905,27 @@ const main = async (): Promise<void> => {
}
const marketTypeStr = getVariant(normedMarketType);
if (useRedis) {
const redisClient = (
marketTypeStr === 'spot' ? spotMarketRedisMap : perpMarketRedisMap
).get(normedMarketIndex).client;
const redisL3 = await redisClient.getRaw(
`last_update_orderbook_l3_${marketTypeStr}_${normedMarketIndex}`
);
if (
redisL3 &&
dlobProvider.getSlot() - parseInt(JSON.parse(redisL3).slot) <
SLOT_STALENESS_TOLERANCE
) {
cacheHitCounter.add(1, {
miss: false,
path: req.baseUrl + req.path,
});
res.writeHead(200);
res.end(redisL3);
return;
} else {
cacheHitCounter.add(1, {
miss: true,
path: req.baseUrl + req.path,
});
}
const redisL3 = await fetchFromRedis(
`last_update_orderbook_l3_${marketTypeStr}_${normedMarketIndex}`,
selectMostRecentBySlot
);
if (
redisL3 &&
dlobProvider.getSlot() - redisL3['slot'] < SLOT_STALENESS_TOLERANCE
) {
cacheHitCounter.add(1, {
miss: false,
path: req.baseUrl + req.path,
});
res.writeHead(200);
res.end(JSON.stringify(redisL3));
return;
} else {
cacheHitCounter.add(1, {
miss: true,
path: req.baseUrl + req.path,
});
}
const l3 = dlobSubscriber.getL3({

View File

@@ -20,7 +20,6 @@ import {
PhoenixSubscriber,
MarketType,
OraclePriceData,
OrderSubscriberConfig,
} from '@drift-labs/sdk';
import { RedisClient, RedisClientPrefix } from '@drift/common/clients';

View File

@@ -79,7 +79,9 @@ const main = async () => {
const redisClient = new RedisClient({ prefix: redisClientPrefix });
await redisClient.connect();
const slotSubscriber = new SlotSubscriber(connection, {});
const slotSubscriber = new SlotSubscriber(connection, {
resubTimeoutMs: 10_000,
});
const lamportsBalance = await connection.getBalance(wallet.publicKey);
logger.info(

View File

@@ -472,3 +472,18 @@ export type SubscriberLookup = {
openbook?: OpenbookV2Subscriber;
};
};
export const selectMostRecentBySlot = (responses: any[]): any => {
const parsedResponses = responses
.map((response) => {
try {
return JSON.parse(response);
} catch {
return null;
}
})
.filter((parsed) => parsed && typeof parsed.slot === 'number');
return parsedResponses.reduce((mostRecent, current) => {
return !mostRecent || current.slot > mostRecent.slot ? current : mostRecent;
}, null);
};

View File

@@ -3,8 +3,8 @@ import express from 'express';
import * as http from 'http';
import compression from 'compression';
import { WebSocket, WebSocketServer } from 'ws';
import { sleep } from './utils/utils';
import { register, Gauge } from 'prom-client';
import { sleep, selectMostRecentBySlot } from './utils/utils';
import { register, Gauge, Counter } from 'prom-client';
import { DriftEnv, PerpMarkets, SpotMarkets } from '@drift-labs/sdk';
import { RedisClient, RedisClientPrefix } from '@drift/common/clients';
@@ -21,6 +21,16 @@ const wsConnectionsGauge = new Gauge({
name: 'websocket_connections',
help: 'Number of active WebSocket connections',
});
const wsOrderbookSourceCounter = new Counter({
name: 'websocket_orderbook_source',
help: 'Number of orderbook messages sent from source',
labelNames: ['source'],
});
const wsOrderbookSourceLastSlotGauge = new Gauge({
name: 'websocket_orderbook_source_last_slot',
help: 'Last slot of orderbook messages from a source',
labelNames: ['source'],
});
const server = http.createServer(app);
const wss = new WebSocketServer({
@@ -34,15 +44,27 @@ const WS_PORT = process.env.WS_PORT || '3000';
console.log(`WS LISTENER PORT : ${WS_PORT}`);
const MAX_BUFFERED_AMOUNT = 300000;
const REDIS_CLIENT = process.env.REDIS_CLIENT || 'DLOB';
const CHANNEL_PREFIX = RedisClientPrefix[REDIS_CLIENT];
console.log('Redis Clients:', REDIS_CLIENT);
const CHANNEL_PREFIX_HELIUS = RedisClientPrefix.DLOB_HELIUS;
const envClients = [];
const clients = process.env.REDIS_CLIENT?.trim()
.replace(/^\[|\]$/g, '')
.split(/\s*,\s*/);
clients?.forEach((client) => envClients.push(RedisClientPrefix[client]));
const REDIS_CLIENTS = envClients.length
? envClients
: [RedisClientPrefix.DLOB, RedisClientPrefix.DLOB_HELIUS];
console.log('Redis Clients:', REDIS_CLIENTS);
const regexp = new RegExp(REDIS_CLIENTS.join('|'), 'g');
const sanitiseChannelForClient = (channel: string | undefined): string => {
return channel
?.replace(CHANNEL_PREFIX, '')
?.replace(CHANNEL_PREFIX_HELIUS, '');
return channel?.replace(regexp, '');
};
const getChannelPrefix = (channel: string | undefined): string | undefined => {
if (!channel) return undefined;
const match = channel.match(regexp);
return match?.[0];
};
const getRedisChannelFromMessage = (message: any): string => {
@@ -70,11 +92,11 @@ const getRedisChannelFromMessage = (message: any): string => {
switch (channel.toLowerCase()) {
case 'trades':
return `${CHANNEL_PREFIX}trades_${marketType}_${marketIndex}`;
return `trades_${marketType}_${marketIndex}`;
case 'orderbook':
return `${CHANNEL_PREFIX}orderbook_${marketType}_${marketIndex}`;
return `orderbook_${marketType}_${marketIndex}`;
case 'priorityfees':
return `${CHANNEL_PREFIX_HELIUS}priorityFees_${marketType}_${marketIndex}`;
return `priorityFees_${marketType}_${marketIndex}`;
case undefined:
default:
throw new Error('Bad channel specified');
@@ -82,47 +104,71 @@ const getRedisChannelFromMessage = (message: any): string => {
};
async function main() {
const redisClient = new RedisClient({});
const lastMessageRetriever = new RedisClient({ prefix: CHANNEL_PREFIX });
const subscribedChannelToSlot: Map<string, number> = new Map();
await redisClient.connect();
await lastMessageRetriever.connect();
const redisClients: Array<RedisClient> = [];
for (let i = 0; i < REDIS_CLIENTS.length; i++) {
const redisClient = new RedisClient({ prefix: REDIS_CLIENTS[i] });
redisClients.push(redisClient);
redisClient.forceGetClient().on('connect', () => {
subscribedChannels.forEach(async (channel) => {
try {
await redisClient.subscribe(channel);
} catch (error) {
console.error(`Error subscribing to ${channel}:`, error);
}
});
});
redisClient.forceGetClient().on('message', (subscribedChannel, message) => {
const sanitizedChannel = sanitiseChannelForClient(subscribedChannel);
const channelPrefix = getChannelPrefix(sanitizedChannel);
const subscribers = channelSubscribers.get(sanitizedChannel);
if (subscribers) {
if (sanitizedChannel.includes('orderbook')) {
const messageSlot = JSON.parse(message)['slot'];
wsOrderbookSourceLastSlotGauge.set(
{
source: channelPrefix,
},
messageSlot
);
const lastMessageSlot = subscribedChannelToSlot.get(sanitizedChannel);
if (!lastMessageSlot || lastMessageSlot <= messageSlot) {
subscribedChannelToSlot.set(sanitizedChannel, messageSlot);
} else if (lastMessageSlot > messageSlot) {
return;
}
}
subscribers.forEach((ws) => {
if (
ws.readyState === WebSocket.OPEN &&
ws.bufferedAmount < MAX_BUFFERED_AMOUNT
) {
wsOrderbookSourceCounter.inc({
source: channelPrefix,
});
ws.send(
JSON.stringify({
channel: sanitizedChannel,
data: message,
})
);
}
});
}
});
redisClient.forceGetClient().on('error', (error) => {
console.error('Redis client error:', error);
});
}
const channelSubscribers = new Map<string, Set<WebSocket>>();
const subscribedChannels = new Set<string>();
redisClient.forceGetClient().on('connect', () => {
subscribedChannels.forEach(async (channel) => {
try {
await redisClient.subscribe(channel);
} catch (error) {
console.error(`Error subscribing to ${channel}:`, error);
}
});
});
redisClient.forceGetClient().on('message', (subscribedChannel, message) => {
const subscribers = channelSubscribers.get(subscribedChannel);
if (subscribers) {
subscribers.forEach((ws) => {
if (
ws.readyState === WebSocket.OPEN &&
ws.bufferedAmount < MAX_BUFFERED_AMOUNT
)
ws.send(
JSON.stringify({
channel: sanitiseChannelForClient(subscribedChannel),
data: message,
})
);
});
}
});
redisClient.forceGetClient().on('error', (error) => {
console.error('Redis client error:', error);
});
wss.on('connection', (ws: WebSocket) => {
console.log('Client connected');
wsConnectionsGauge.inc();
@@ -166,8 +212,12 @@ async function main() {
if (!subscribedChannels.has(redisChannel)) {
console.log('Trying to subscribe to channel', redisChannel);
redisClient
.subscribe(redisChannel)
redisClients[0]
.subscribe(
redisChannel.includes('priorityFees')
? `dlob-helius:${redisChannel}`
: `${redisClients[0].getPrefix()}${redisChannel}`
)
.then(() => {
subscribedChannels.add(redisChannel);
})
@@ -179,6 +229,21 @@ async function main() {
);
return;
});
if (redisChannel.includes('orderbook') && redisClients.length > 1) {
redisClients.slice(1).map((redisClient) => {
redisClient
.subscribe(`${redisClient.getPrefix()}${redisChannel}`)
.catch(() => {
ws.send(
JSON.stringify({
error: `Error subscribing to channel: ${parsedMessage}`,
})
);
return;
});
});
}
}
if (!channelSubscribers.get(redisChannel)) {
@@ -194,17 +259,21 @@ async function main() {
);
// Fetch and send last message
if (redisChannel.includes('orderbook')) {
const lastUpdateChannel = `last_update_${redisChannel}`.replace(
CHANNEL_PREFIX,
''
);
const lastMessage = await lastMessageRetriever.getRaw(
lastUpdateChannel
const lastMessages = await Promise.all(
redisClients.map((redisClient) =>
redisClient.getRaw(
`last_update_${redisChannel}`.replace(
redisClient.getPrefix(),
''
)
)
)
);
const lastMessage = selectMostRecentBySlot(lastMessages);
if (lastMessage) {
ws.send(
JSON.stringify({
channel: lastUpdateChannel,
channel: redisChannel,
data: lastMessage,
})
);
@@ -278,7 +347,7 @@ async function main() {
clearInterval(bufferInterval);
channelSubscribers.forEach((subscribers, channel) => {
if (subscribers.delete(ws) && subscribers.size === 0) {
redisClient.unsubscribe(channel);
redisClients.map((redisClient) => redisClient.unsubscribe(channel));
channelSubscribers.delete(channel);
subscribedChannels.delete(channel);
}