trade/trade-drift-dlob
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #234 from drift-labs/master

Browse Source 
bump common + nick's script changes
This commit is contained in:
moosecat
2024-08-13 13:12:09 -07:00
committed by GitHub
parent 93f7fdada9 219e4f5e0c
commit 0c0b20a3db
4 changed files with 195 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
drift-common

Submodule drift-common updated: ec8383719d...82dbbf336b

149
package.json
View File

@@ -1,75 +1,76 @@
{ {
"name": "@drift-labs/dlob-server", "name": "@drift-labs/dlob-server",
"version": "0.1.0", "version": "0.1.0",
"author": "wphan", "author": "wphan",
"main": "lib/index.js", "main": "lib/index.js",
"license": "Apache-2.0", "license": "Apache-2.0",
"dependencies": { "dependencies": {
"@coral-xyz/anchor": "^0.29.0", "@coral-xyz/anchor": "^0.29.0",
"@drift/common": "file:./drift-common/common-ts", "@drift/common": "file:./drift-common/common-ts",
"@drift-labs/sdk": "file:./drift-common/protocol/sdk", "@drift-labs/sdk": "file:./drift-common/protocol/sdk",
"@opentelemetry/api": "^1.1.0", "@opentelemetry/api": "^1.1.0",
"@opentelemetry/auto-instrumentations-node": "^0.31.1", "@opentelemetry/auto-instrumentations-node": "^0.31.1",
"@opentelemetry/exporter-prometheus": "^0.31.0", "@opentelemetry/exporter-prometheus": "^0.31.0",
"@opentelemetry/sdk-node": "^0.31.0", "@opentelemetry/sdk-node": "^0.31.0",
"@project-serum/anchor": "^0.19.1-beta.1", "@project-serum/anchor": "^0.19.1-beta.1",
"@project-serum/serum": "^0.13.65", "@project-serum/serum": "^0.13.65",
"@solana/web3.js": "^1.73.3", "@solana/web3.js": "^1.73.3",
"@triton-one/yellowstone-grpc": "^0.3.0", "@triton-one/yellowstone-grpc": "^0.3.0",
"@types/redis": "^4.0.11", "@types/redis": "^4.0.11",
"@types/ws": "^8.5.8", "@types/ws": "^8.5.8",
"async-mutex": "^0.4.0", "async-mutex": "^0.4.0",
"axios": "^1.6.2", "axios": "^1.6.2",
"commander": "^9.4.0", "bottleneck": "^2.19.5",
"compression": "^1.7.4", "commander": "^9.4.0",
"cors": "^2.8.5", "compression": "^1.7.4",
"dotenv": "^10.0.0", "cors": "^2.8.5",
"express": "^4.18.2", "dotenv": "^10.0.0",
"ioredis": "^5.3.2", "express": "^4.18.2",
"morgan": "^1.10.0", "ioredis": "^5.3.2",
"prom-client": "^15.0.0", "morgan": "^1.10.0",
"redis": "^4.6.10", "prom-client": "^15.0.0",
"response-time": "^2.3.2", "redis": "^4.6.10",
"rxjs": "^7.8.1", "response-time": "^2.3.2",
"socket.io-redis": "^6.1.1", "rxjs": "^7.8.1",
"typescript": "4.5.4", "socket.io-redis": "^6.1.1",
"undici": "^6.16.1", "typescript": "4.5.4",
"winston": "^3.8.1", "undici": "^6.16.1",
"ws": "^8.14.2" "winston": "^3.8.1",
}, "ws": "^8.14.2"
"devDependencies": { },
"@types/k6": "^0.45.0", "devDependencies": {
"@typescript-eslint/eslint-plugin": "^4.28.0", "@types/k6": "^0.45.0",
"@typescript-eslint/parser": "^4.28.0", "@typescript-eslint/eslint-plugin": "^4.28.0",
"eslint": "^7.29.0", "@typescript-eslint/parser": "^4.28.0",
"eslint-config-prettier": "^8.3.0", "eslint": "^7.29.0",
"eslint-plugin-prettier": "^3.4.0", "eslint-config-prettier": "^8.3.0",
"husky": "^7.0.4", "eslint-plugin-prettier": "^3.4.0",
"k6": "^0.0.0", "husky": "^7.0.4",
"prettier": "^2.4.1", "k6": "^0.0.0",
"ts-node": "^10.9.1" "prettier": "^2.4.1",
}, "ts-node": "^10.9.1"
"scripts": { },
"prepare": "husky install", "scripts": {
"build": "yarn clean && tsc", "prepare": "husky install",
"clean": "rm -rf lib", "build": "yarn clean && tsc",
"start": "node lib/index.js", "clean": "rm -rf lib",
"dev": "ts-node src/index.ts", "start": "node lib/index.js",
"server-lite": "ts-node src/serverLite.ts", "dev": "ts-node src/index.ts",
"dlob-publish": "ts-node src/publishers/dlobPublisher.ts", "server-lite": "ts-node src/serverLite.ts",
"trades-publish": "ts-node src/publishers/tradesPublisher.ts", "dlob-publish": "ts-node src/publishers/dlobPublisher.ts",
"fees-publish": "ts-node src/publishers/priorityFeesPublisher.ts", "trades-publish": "ts-node src/publishers/tradesPublisher.ts",
"pnl-publish": "ts-node src/scripts/publishUnsettledPnlUsers.ts", "fees-publish": "ts-node src/publishers/priorityFeesPublisher.ts",
"ws-manager": "ts-node src/wsConnectionManager.ts", "pnl-publish": "ts-node src/scripts/publishUnsettledPnlUsers.ts",
"ws-manager:inspect": "yarn build && node --inspect ./lib/wsConnectionManager.js", "ws-manager": "ts-node src/wsConnectionManager.ts",
"dev:inspect": "yarn build && node --inspect ./lib/index.js", "ws-manager:inspect": "yarn build && node --inspect ./lib/wsConnectionManager.js",
"dev:debug": "yarn build && node --inspect-brk --inspect=2230 ./lib/index.js", "dev:inspect": "yarn build && node --inspect ./lib/index.js",
"example": "ts-node example/client.ts", "dev:debug": "yarn build && node --inspect-brk --inspect=2230 ./lib/index.js",
"exampleWithSlot": "ts-node example/clientWithSlot.ts", "example": "ts-node example/client.ts",
"prettify": "prettier --check './src/**/*.ts'", "exampleWithSlot": "ts-node example/clientWithSlot.ts",
"prettify:fix": "prettier --write './src/**/*.ts'", "prettify": "prettier --check './src/**/*.ts'",
"lint": "eslint . --ext ts --quiet", "prettify:fix": "prettier --write './src/**/*.ts'",
"lint:fix": "eslint . --ext ts --fix", "lint": "eslint . --ext ts --quiet",
"playground": "ts-node src/playground.ts" "lint:fix": "eslint . --ext ts --fix",
} "playground": "ts-node src/playground.ts"
} }
}

5
src/index.ts
View File

@@ -657,7 +657,10 @@ const main = async (): Promise<void> => {
return; return;
} }
const redisClient = perpMarketRedisMap.get(marketIndex).client; const redisClient = redisClients.find(
(client) =>
client.forceGetClient().options.keyPrefix === RedisClientPrefix.DLOB
);
const redisResponseGainers = await redisClient.getRaw( const redisResponseGainers = await redisClient.getRaw(
`perp_market_${marketIndex}_gainers` `perp_market_${marketIndex}_gainers`

157
src/scripts/publishUnsettledPnlUsers.ts
View File

@@ -1,4 +1,4 @@
import { RedisClient, RedisClientPrefix } from '@drift/common'; import { RedisClient, RedisClientPrefix, sleep } from '@drift/common';
import { import {
BigNum, BigNum,
DriftClient, DriftClient,
@@ -11,11 +11,11 @@ import {
Wallet, Wallet,
ZERO, ZERO,
calculateClaimablePnl, calculateClaimablePnl,
calculatePositionPNL,
decodeUser, decodeUser,
} from '@drift-labs/sdk'; } from '@drift-labs/sdk';
import { Connection, Keypair } from '@solana/web3.js'; import { Connection, Keypair } from '@solana/web3.js';
import { logger } from '../utils/logger'; import { logger } from '../utils/logger';
import Bottleneck from 'bottleneck';
const dotenv = require('dotenv'); const dotenv = require('dotenv');
dotenv.config(); dotenv.config();
@@ -29,10 +29,19 @@ type UserPnlMap = {
}; };
}; };
type AllPnlUsers = {
[perpMarketIndex: number]: {
users: { userPubKey: string; pnl: number }[];
};
};
const startTime = Date.now(); const startTime = Date.now();
const endpoint = process.env.ENDPOINT as string; const endpoint = process.env.ENDPOINT as string;
const wsEndpoint = process.env.WS_ENDPOINT as string; const wsEndpoint = process.env.WS_ENDPOINT as string;
const driftEnv = process.env.ENV as string; const driftEnv = process.env.ENV as string;
const chunkSize = Number(process.env.CHUNK_SIZE) || 100;
const sleepTimeMs = Number(process.env.SLEEP_TIME_MS) || 500;
const delayMs = Number(process.env.DEFAULT_DELAY_MS) || 100;
const connection = new Connection(endpoint, { const connection = new Connection(endpoint, {
commitment: 'confirmed', commitment: 'confirmed',
@@ -45,6 +54,8 @@ const driftClient = new DriftClient({
}); });
const userMapRedisClient = new RedisClient({ const userMapRedisClient = new RedisClient({
host: process.env.ELASTICACHE_USERMAP_HOST ?? process.env.ELASTICACHE_HOST,
port: process.env.ELASTICACHE_USERMAP_PORT ?? process.env.ELASTICACHE_PORT,
prefix: RedisClientPrefix.USER_MAP, prefix: RedisClientPrefix.USER_MAP,
}); });
@@ -56,14 +67,41 @@ const main = async () => {
// get the users from usermap redis client // get the users from usermap redis client
await driftClient.subscribe(); await driftClient.subscribe();
const limiter = new Bottleneck({
maxConcurrent: 10,
minTime: delayMs,
});
const userStrings = await userMapRedisClient.lRange('user_pubkeys', 0, -1); const userStrings = await userMapRedisClient.lRange('user_pubkeys', 0, -1);
const redisUsers = await Promise.all( const totalCount = userStrings.length;
userStrings.map((userStr) => getUserFromRedis(userStr)) let finishedCount = 0;
); let allNonZeroPnls = {};
// construct an object with the top 20/bottom 20 unsettled in each perp market logger.info(`Starting loop for total of ${totalCount} users`);
const userPnlMap = createMarketSpecificPnlLeaderboards(redisUsers);
while (finishedCount < totalCount) {
const userChunk = userStrings.slice(
finishedCount,
finishedCount + chunkSize
);
limiter.schedule(async () => {
const redisUsers = await Promise.all(
userChunk.map((userStr) => getUserFromRedis(userStr))
);
// add all the nonzero pnl users from each market in chunks
allNonZeroPnls = buildUserMarketLists(redisUsers, allNonZeroPnls);
});
finishedCount += chunkSize;
logger.info(`Wrote ${finishedCount} users, sleeping for ${sleepTimeMs}ms`);
await sleep(sleepTimeMs);
}
const userPnlMap = createMarketPnlLeaderboards(allNonZeroPnls);
const success = await writeToDlobRedis(userPnlMap); const success = await writeToDlobRedis(userPnlMap);
@@ -73,6 +111,12 @@ const main = async () => {
} in ${Date.now() - startTime} ms` } in ${Date.now() - startTime} ms`
); );
await sleep(5000);
logger.info('==== TEST PRINT LOSERS FROM BTC-PERP ====');
const losers = await dlobRedisClient.getRaw(`perp_market_1_losers`);
logger.info(losers);
process.exit(); process.exit();
}; };
@@ -84,30 +128,66 @@ const writeToDlobRedis = async (userPnlMap: UserPnlMap): Promise<boolean> => {
const losersRedisKey = `perp_market_${perpMarketIndex}_losers`; const losersRedisKey = `perp_market_${perpMarketIndex}_losers`;
// write the new lists // write the new lists
await dlobRedisClient.setRaw( await Promise.all([
gainersRedisKey, dlobRedisClient.setRaw(
JSON.stringify(userPnlMap[perpMarketIndex].gain) gainersRedisKey,
); JSON.stringify(userPnlMap[perpMarketIndex].gain ?? [])
await dlobRedisClient.setRaw( ),
losersRedisKey, dlobRedisClient.setRaw(
JSON.stringify(userPnlMap[perpMarketIndex].loss) losersRedisKey,
); JSON.stringify(userPnlMap[perpMarketIndex].loss ?? [])
),
return; ]).then(([_winnersResult, _losersResult]) => {
logger.info(
`Wrote ${userPnlMap[perpMarketIndex].gain.length} users to redis key: ${gainersRedisKey}`
);
logger.info(
`Wrote ${userPnlMap[perpMarketIndex].loss.length} users to redis key: ${losersRedisKey}`
);
});
}) })
); );
return true; return true;
} catch (e) { } catch (e) {
console.log('Error writing to dlob redis client: ', e); logger.info(`Error writing to dlob redis client: ${e}`);
return false; return false;
} }
}; };
const createMarketSpecificPnlLeaderboards = ( const createMarketPnlLeaderboards = (allPnlUsers: AllPnlUsers): UserPnlMap => {
redisUsers: { user: User; bufferString: string }[] let pnlMap = {};
): UserPnlMap => {
const pnlMap = {}; Object.keys(allPnlUsers).forEach((perpMarketIndex) => {
try {
const sortedPnls = allPnlUsers[perpMarketIndex].users.sort(
(a, b) => b.pnl - a.pnl
);
// store the top 20 winners and losers in each perp market
pnlMap[perpMarketIndex] = {
gain: sortedPnls.slice(0, 20),
loss: sortedPnls.slice(-20).reverse(),
};
} catch (e) {
logger.info(
`Error in ${PerpMarkets[driftEnv][perpMarketIndex].symbol}: ${e}`
);
pnlMap[perpMarketIndex] = {
gain: [],
loss: [],
};
}
});
return pnlMap;
};
const buildUserMarketLists = (
redisUsers: { user: User; bufferString: string }[],
allPnlUsers: AllPnlUsers
): AllPnlUsers => {
let newPnlUsers = allPnlUsers;
const usdcSpotMarket = driftClient.getSpotMarketAccount( const usdcSpotMarket = driftClient.getSpotMarketAccount(
QUOTE_SPOT_MARKET_INDEX QUOTE_SPOT_MARKET_INDEX
@@ -123,7 +203,7 @@ const createMarketSpecificPnlLeaderboards = (
perpMarket.marketIndex perpMarket.marketIndex
); );
const allNonZeroPnls = redisUsers.flatMap((redisUser) => { const allNonZeroPnlsInMarket = redisUsers.flatMap((redisUser) => {
try { try {
const perpPosition = redisUser.user.getPerpPosition( const perpPosition = redisUser.user.getPerpPosition(
perpMarket.marketIndex perpMarket.marketIndex
@@ -144,22 +224,13 @@ const createMarketSpecificPnlLeaderboards = (
false false
)[0]; )[0];
let marketPnl = calculatePositionPNL( const marketPnl = calculateClaimablePnl(
perpMarketAccount, perpMarketAccount,
usdcSpotMarket,
perpPositionWithLpSettle, perpPositionWithLpSettle,
true,
oraclePriceData oraclePriceData
); );
if (marketPnl.gt(ZERO)) {
marketPnl = calculateClaimablePnl(
perpMarketAccount,
usdcSpotMarket,
perpPositionWithLpSettle,
oraclePriceData
);
}
return marketPnl.eq(ZERO) return marketPnl.eq(ZERO)
? [] ? []
: { : {
@@ -175,19 +246,21 @@ const createMarketSpecificPnlLeaderboards = (
} }
}); });
const sortedPnls = allNonZeroPnls.sort((a, b) => b.pnl - a.pnl); if (!allPnlUsers[perpMarket.marketIndex]) {
newPnlUsers[perpMarket.marketIndex] = { users: allNonZeroPnlsInMarket };
// store the top 20 winners and losers in each perp market } else {
pnlMap[perpMarket.marketIndex] = { newPnlUsers[perpMarket.marketIndex] = {
gain: sortedPnls.slice(0, 20), users: allPnlUsers[perpMarket.marketIndex].users.concat(
loss: sortedPnls.slice(-20, -1), allNonZeroPnlsInMarket
}; ),
};
}
} catch (e) { } catch (e) {
logger.error(`Could not fetch PnLs for ${perpMarket.symbol}: `, e); logger.error(`Could not fetch PnLs for ${perpMarket.symbol}: `, e);
} }
} }
return pnlMap; return newPnlUsers;
}; };
const getUserFromRedis = async (userAccountStr: string) => { const getUserFromRedis = async (userAccountStr: string) => {