trade/trade-drift-dlob
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #537 from drift-labs/wphan/auc-params-v2

Browse Source 
Wphan/auc params v2
This commit is contained in:
wphan
2025-10-22 13:48:23 -07:00
committed by GitHub
parent 6a02ba4c21 2426720fbc
commit d6097b974b
9 changed files with 1761 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@@ -5,6 +5,7 @@
"main": "lib/index.js",
"license": "Apache-2.0",
"dependencies": {
"@aws-sdk/client-athena": "^3.830.0",
"@aws-sdk/client-kinesis": "^3.830.0",
"@aws-sdk/util-retry": "^3.374.0",
"@coral-xyz/anchor": "^0.29.0",

237
src/athena/client.ts Normal file
View File

@@ -0,0 +1,237 @@
import {
AthenaClient,
ColumnInfo,
GetQueryExecutionCommand,
GetQueryResultsCommand,
GetQueryResultsCommandOutput,
QueryExecutionState,
Row,
StartQueryExecutionCommand,
StartQueryExecutionCommandInput,
} from '@aws-sdk/client-athena';
import { ConfiguredRetryStrategy } from '@aws-sdk/util-retry';
import Bottleneck from 'bottleneck';
import { logger } from '../utils/logger';
const QUERY_TIMEOUT = 300000;
const POLL_INTERVAL = 1000;
const DEFAULT_ATHENA_DATABASE = 'staging-archive';
const DEFAULT_ATHENA_OUTPUT_BUCKET = 'staging-data-ingestion-bucket';
const limiter = new Bottleneck({
maxConcurrent: 5,
minTime: 100,
});
type QueryResult = Record<string, string | null>;
const parseRow = (row: Row, columnInfo: ColumnInfo[]): QueryResult => {
const rowData = row.Data;
if (!rowData) {
return {};
}
return columnInfo.reduce((acc: QueryResult, column, index) => {
if (column.Name) {
acc[column.Name] = rowData[index]?.VarCharValue ?? null;
}
return acc;
}, {});
};
const athena = new AthenaClient({
region: process.env.AWS_REGION,
retryStrategy: new ConfiguredRetryStrategy(
3,
(attempt: number) => 100 + attempt * 1000
),
});
export const Athena = ({
overrideDatabaseName,
overrideBucketName,
}: { overrideDatabaseName?: string; overrideBucketName?: string } = {}) => {
const database =
overrideDatabaseName ??
process.env.ATHENA_DATABASE ??
DEFAULT_ATHENA_DATABASE;
const outputLocation = `s3://${
overrideBucketName ??
process.env.ATHENA_OUTPUT_BUCKET ??
DEFAULT_ATHENA_OUTPUT_BUCKET
}/athena`;
const startQuery = async (query: string, params?: Record<string, string>) => {
const executionParams: StartQueryExecutionCommandInput = {
QueryString: query,
QueryExecutionContext: {
Database: database,
},
ResultConfiguration: {
OutputLocation: outputLocation,
},
ResultReuseConfiguration: {
ResultReuseByAgeConfiguration: {
Enabled: false,
},
},
};
if (params) {
executionParams.ExecutionParameters = Object.values(params);
}
const { QueryExecutionId } = await athena.send(
new StartQueryExecutionCommand(executionParams)
);
if (!QueryExecutionId) {
throw new Error('Failed to start query execution');
}
return QueryExecutionId;
};
const getQueryExecution = async (queryExecutionId: string) => {
const { QueryExecution } = await athena.send(
new GetQueryExecutionCommand({
QueryExecutionId: queryExecutionId,
})
);
return QueryExecution;
};
const waitForQueryCompletion = async (
queryExecutionId: string,
timeout: number = QUERY_TIMEOUT
): Promise<void> => {
const startTime = Date.now();
while (Date.now() - startTime < timeout) {
const execution = await getQueryExecution(queryExecutionId);
logger.info(
`Total bytes scanned: ${
(execution?.Statistics?.DataScannedInBytes ?? 0) / 1024
}kb`
);
const state = execution?.Status?.State;
switch (state) {
case QueryExecutionState.SUCCEEDED:
return;
case QueryExecutionState.FAILED:
throw new Error(
`Query failed: ${execution?.Status?.StateChangeReason}`
);
case QueryExecutionState.CANCELLED:
throw new Error('Query was cancelled');
default:
await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL));
}
}
throw new Error(`Query timeout after ${timeout}ms`);
};
const getQueryResults = async (
queryExecutionId: string,
nextToken?: string
): Promise<GetQueryResultsCommandOutput> => {
return athena.send(
new GetQueryResultsCommand({
QueryExecutionId: queryExecutionId,
NextToken: nextToken,
})
);
};
const getAllQueryResults = async (
queryExecutionId: string
): Promise<QueryResult[]> => {
let nextToken: string | undefined;
const allResults: QueryResult[] = [];
do {
const results = await getQueryResults(queryExecutionId, nextToken);
if (
results.ResultSet?.Rows &&
results.ResultSet.ResultSetMetadata?.ColumnInfo
) {
const columnInfo = results.ResultSet.ResultSetMetadata.ColumnInfo;
const rows = nextToken
? results.ResultSet.Rows
: results.ResultSet.Rows.slice(1);
const parsedRows = rows.map((row) => parseRow(row, columnInfo));
allResults.push(...parsedRows);
}
nextToken = results.NextToken;
} while (nextToken);
return allResults;
};
const query = async (
queryString: string,
params?: Record<string, string>
): Promise<QueryResult[]> => {
try {
const queryExecutionId = await startQuery(queryString, params);
await waitForQueryCompletion(queryExecutionId);
return getAllQueryResults(queryExecutionId);
} catch (error) {
const { message } = error as Error;
logger.error(`Error executing Athena query: ${message}`);
throw error;
}
};
const batchQuery = async ({
queries,
}: {
queries: { query: string; params?: Record<string, string> }[];
}): Promise<{
results: QueryResult[][];
errors: Error[];
}> => {
const results: QueryResult[][] = [];
const errors: Error[] = [];
await Promise.all(
queries.map(async ({ query: queryString, params }, index) => {
try {
const result = await limiter.schedule(() =>
query(queryString, params)
);
results[index] = result;
} catch (error) {
logger.error(
`Error in batch query ${index}: ${queryString}, Error: ${error}`
);
errors[index] = error as Error;
}
})
);
if (errors.length > 0) {
logger.warn(`${errors.length} queries failed in batch execution`);
}
return {
results,
errors,
};
};
return {
query,
batchQuery,
startQuery,
getQueryExecution,
getQueryResults,
getAllQueryResults,
waitForQueryCompletion,
};
};

3
src/athena/index.ts Normal file
View File

@@ -0,0 +1,3 @@
export * from './client';
export * from './utils';
export * from './repositories/fillQualityAnalytics';

298
src/athena/repositories/fillQualityAnalytics.ts Normal file
View File

@@ -0,0 +1,298 @@
import { Athena } from '../client';
export interface TakerFillVsOracleBpsResult {
MarketIndex: string;
TakerBuyBpsFromOracle_ALL: string | null;
TakerSellBpsFromOracle_ALL: string | null;
TakerBuyBpsFromOracle_1e0: string | null;
TakerBuyBpsFromOracle_1e3: string | null;
TakerBuyBpsFromOracle_1e4: string | null;
TakerBuyBpsFromOracle_1e5: string | null;
TakerBuyBpsFromOracle_1e6: string | null;
TakerSellBpsFromOracle_1e0: string | null;
TakerSellBpsFromOracle_1e3: string | null;
TakerSellBpsFromOracle_1e4: string | null;
TakerSellBpsFromOracle_1e5: string | null;
TakerSellBpsFromOracle_1e6: string | null;
Zero: string;
}
export type TakerFillVsOracleBpsRedisResult = {
marketIndex: string;
takerBuyBpsFromOracle: {
all: string | null;
'1e0': string | null;
'1e3': string | null;
'1e4': string | null;
'1e5': string | null;
'1e6': string | null;
};
takerSellBpsFromOracle: {
all: string | null;
'1e0': string | null;
'1e3': string | null;
'1e4': string | null;
'1e5': string | null;
'1e6': string | null;
};
updatedAtTs: number;
};
export const FillQualityAnalyticsRepository = () => {
const { query } = Athena();
/**
* Get taker fill vs oracle basis points, bucketed by order notional cohorts at place-time size.
* Returns a rolling average per cohort/direction, partitioned by market.
* This query returns data for ALL perp markets in a single result set.
*
* @param from - Unix timestamp in milliseconds
* @param to - Unix timestamp in milliseconds
* @param baseDecimals - Base decimals for the market (default: 9)
* @param smoothingMinutes - Minutes for rolling average (default: 60)
* @returns Array of time series data with taker fill vs oracle bps by cohort and market
*/
const getTakerFillVsOracleBps = async (
fromMs: number,
toMs: number,
baseDecimals: number = 9,
smoothingMinutes: number = 60
): Promise<TakerFillVsOracleBpsResult[]> => {
// Taker fill vs Oracle bps, bucketed by order notional cohorts at place-time size
// Final output: latest non-null rolling average per cohort/direction, PARTITIONED BY market
const queryString = `
WITH
from_dt AS (
SELECT
date_format(from_unixtime(CAST(${fromMs}/1000 AS BIGINT)) AT TIME ZONE 'UTC','%Y') AS yf,
date_format(from_unixtime(CAST(${fromMs}/1000 AS BIGINT)) AT TIME ZONE 'UTC','%m') AS mf,
date_format(from_unixtime(CAST(${fromMs}/1000 AS BIGINT)) AT TIME ZONE 'UTC','%d') AS df
),
to_dt AS (
SELECT
date_format(from_unixtime(CAST(${toMs}/1000 AS BIGINT)) AT TIME ZONE 'UTC','%Y') AS yt,
date_format(from_unixtime(CAST(${toMs}/1000 AS BIGINT)) AT TIME ZONE 'UTC','%m') AS mt,
date_format(from_unixtime(CAST(${toMs}/1000 AS BIGINT)) AT TIME ZONE 'UTC','%d') AS dt
),
-- Dedup to the earliest OrderRecord per (txsig,user,orderid,marketindex) to reflect "place" state
orders_dedup AS (
SELECT *
FROM (
SELECT
CAST(orx.ts AS BIGINT) AS order_ts_epoch,
orx.txsig AS txsig,
orx.user AS user,
orx."order".orderid AS orderid,
orx."order".marketindex AS marketindex,
CAST(orx."order".baseassetamount AS DOUBLE) AS base_asset_amount_raw,
ROW_NUMBER() OVER (
PARTITION BY orx.txsig, orx.user, orx."order".orderid, orx."order".marketindex
ORDER BY CAST(orx.ts AS BIGINT) ASC
) AS rn
FROM eventtype_orderrecord orx
CROSS JOIN from_dt f
CROSS JOIN to_dt t
WHERE
orx."order".markettype = 'perp'
-- UTC-safe pruning on VARCHAR partitions
AND orx.year BETWEEN f.yf AND t.yt
AND (
orx.year > f.yf
OR (orx.year = f.yf AND orx.month > f.mf)
OR (orx.year = f.yf AND orx.month = f.mf AND orx.day >= f.df)
)
AND (
orx.year < t.yt
OR (orx.year = t.yt AND orx.month < t.mt)
OR (orx.year = t.yt AND orx.month = t.mt AND orx.day <= t.dt)
)
-- absolute time guardrails (ts is epoch seconds in table)
AND CAST(orx.ts AS BIGINT) BETWEEN CAST(${fromMs}/1000 AS BIGINT) AND CAST(${toMs}/1000 AS BIGINT)
) t1
WHERE rn = 1
),
-- Trade fills (taker perspective) within the window (all markets)
trades AS (
SELECT
CAST(tr.ts AS BIGINT) AS ts_epoch,
tr.txsig AS txsig,
tr.taker AS user,
tr.takerorderid AS orderid,
tr.marketindex AS marketindex,
LOWER(tr.takerorderdirection) AS dir,
CAST(tr.quoteassetamountfilled AS DOUBLE) AS q_filled,
CAST(tr.baseassetamountfilled AS DOUBLE) AS b_filled,
CAST(tr.oracleprice AS DOUBLE) AS oracle_raw
FROM eventtype_traderecord tr
CROSS JOIN from_dt f
CROSS JOIN to_dt t
WHERE
LOWER(tr.action) = 'fill'
AND tr.markettype = 'perp'
AND LOWER(tr.takerorderdirection) IN ('long','short')
-- UTC-safe pruning on VARCHAR partitions
AND tr.year BETWEEN f.yf AND t.yt
AND (
tr.year > f.yf
OR (tr.year = f.yf AND tr.month > f.mf)
OR (tr.year = f.yf AND tr.month = f.mf AND tr.day >= f.df)
)
AND (
tr.year < t.yt
OR (tr.year = t.yt AND tr.month < t.mt)
OR (tr.year = t.yt AND tr.month = t.mt AND tr.day <= t.dt)
)
-- absolute time guardrails (ts is epoch seconds in table)
AND CAST(tr.ts AS BIGINT) BETWEEN CAST(${fromMs}/1000 AS BIGINT) AND CAST(${toMs}/1000 AS BIGINT)
),
-- Join fills to the order's initial size; compute notional cohort and taker bps from oracle
joined AS (
SELECT
from_unixtime(tr.ts_epoch) AS Time,
tr.marketindex AS MarketIndex,
tr.dir AS dir,
(
(
(tr.q_filled / NULLIF(tr.b_filled, 0))
* pow(10.0, (${baseDecimals} - 6))
)
/ (tr.oracle_raw / 1e6) - 1.0
) * 10000.0 AS taker_bps_from_oracle,
( (od.base_asset_amount_raw * 1e-9) * (tr.oracle_raw * 1e-6) ) AS order_value
FROM trades tr
JOIN orders_dedup od
ON od.txsig = tr.txsig
AND od.user = tr.user
AND od.orderid = tr.orderid
AND od.marketindex = tr.marketindex
),
-- Bucket into cohorts
bucketed AS (
SELECT
Time,
MarketIndex,
dir,
taker_bps_from_oracle,
CASE
WHEN order_value > 0 AND order_value < 1000 THEN '1e0'
WHEN order_value >= 1000 AND order_value < 10000 THEN '1e3'
WHEN order_value >= 10000 AND order_value < 100000 THEN '1e4'
WHEN order_value >= 100000 AND order_value < 1000000 THEN '1e5'
WHEN order_value >= 1000000 THEN '1e6'
ELSE 'other'
END AS cohort
FROM joined
),
-- Compute rolling averages
rolling_avgs AS (
SELECT
Time,
MarketIndex,
-- Non-cohort series (ALL fills regardless of cohort)
AVG(CASE WHEN dir = 'long' THEN taker_bps_from_oracle END)
OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
AS "TakerBuyBpsFromOracle_ALL",
AVG(CASE WHEN dir = 'short' THEN taker_bps_from_oracle END)
OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
AS "TakerSellBpsFromOracle_ALL",
-- rolling average by cohort/direction (NULLs ignored by AVG)
AVG(CASE WHEN dir = 'long' AND cohort = '1e0' THEN taker_bps_from_oracle END)
OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
AS "TakerBuyBpsFromOracle_1e0",
AVG(CASE WHEN dir = 'long' AND cohort = '1e3' THEN taker_bps_from_oracle END)
OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
AS "TakerBuyBpsFromOracle_1e3",
AVG(CASE WHEN dir = 'long' AND cohort = '1e4' THEN taker_bps_from_oracle END)
OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
AS "TakerBuyBpsFromOracle_1e4",
AVG(CASE WHEN dir = 'long' AND cohort = '1e5' THEN taker_bps_from_oracle END)
OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
AS "TakerBuyBpsFromOracle_1e5",
AVG(CASE WHEN dir = 'long' AND cohort = '1e6' THEN taker_bps_from_oracle END)
OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
AS "TakerBuyBpsFromOracle_1e6",
AVG(CASE WHEN dir = 'short' AND cohort = '1e0' THEN taker_bps_from_oracle END)
OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
AS "TakerSellBpsFromOracle_1e0",
AVG(CASE WHEN dir = 'short' AND cohort = '1e3' THEN taker_bps_from_oracle END)
OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
AS "TakerSellBpsFromOracle_1e3",
AVG(CASE WHEN dir = 'short' AND cohort = '1e4' THEN taker_bps_from_oracle END)
OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
AS "TakerSellBpsFromOracle_1e4",
AVG(CASE WHEN dir = 'short' AND cohort = '1e5' THEN taker_bps_from_oracle END)
OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
AS "TakerSellBpsFromOracle_1e5",
AVG(CASE WHEN dir = 'short' AND cohort = '1e6' THEN taker_bps_from_oracle END)
OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
AS "TakerSellBpsFromOracle_1e6",
ROW_NUMBER() OVER (PARTITION BY MarketIndex ORDER BY Time DESC) AS rn
FROM bucketed
WHERE taker_bps_from_oracle IS NOT NULL
)
-- Select only the latest row per market
SELECT
Time,
MarketIndex,
"TakerBuyBpsFromOracle_ALL",
"TakerSellBpsFromOracle_ALL",
"TakerBuyBpsFromOracle_1e0",
"TakerBuyBpsFromOracle_1e3",
"TakerBuyBpsFromOracle_1e4",
"TakerBuyBpsFromOracle_1e5",
"TakerBuyBpsFromOracle_1e6",
"TakerSellBpsFromOracle_1e0",
"TakerSellBpsFromOracle_1e3",
"TakerSellBpsFromOracle_1e4",
"TakerSellBpsFromOracle_1e5",
"TakerSellBpsFromOracle_1e6"
FROM rolling_avgs
WHERE rn = 1
ORDER BY MarketIndex;
`;
const results = await query(queryString);
return results.map((result) => ({
MarketIndex: result.MarketIndex || '',
TakerBuyBpsFromOracle_ALL: result.TakerBuyBpsFromOracle_ALL,
TakerSellBpsFromOracle_ALL: result.TakerSellBpsFromOracle_ALL,
TakerBuyBpsFromOracle_1e0: result.TakerBuyBpsFromOracle_1e0,
TakerBuyBpsFromOracle_1e3: result.TakerBuyBpsFromOracle_1e3,
TakerBuyBpsFromOracle_1e4: result.TakerBuyBpsFromOracle_1e4,
TakerBuyBpsFromOracle_1e5: result.TakerBuyBpsFromOracle_1e5,
TakerBuyBpsFromOracle_1e6: result.TakerBuyBpsFromOracle_1e6,
TakerSellBpsFromOracle_1e0: result.TakerSellBpsFromOracle_1e0,
TakerSellBpsFromOracle_1e3: result.TakerSellBpsFromOracle_1e3,
TakerSellBpsFromOracle_1e4: result.TakerSellBpsFromOracle_1e4,
TakerSellBpsFromOracle_1e5: result.TakerSellBpsFromOracle_1e5,
TakerSellBpsFromOracle_1e6: result.TakerSellBpsFromOracle_1e6,
Zero: result.Zero || '0',
}));
};
return {
getTakerFillVsOracleBps,
};
};

39
src/athena/utils.ts Normal file
View File

@@ -0,0 +1,39 @@
export const getTimePartition = (from: number, to: number) => {
return `WITH time_range AS (
SELECT
${from} as from_ts,
${to} as to_ts,
DATE_FORMAT(from_unixtime(${from}), '%Y%m%d') as from_date,
DATE_FORMAT(from_unixtime(${to}), '%Y%m%d') as to_date
)`;
};
export const getTimeRangeAndPartitions = (
from: number,
to: number,
valid_partitions_label = 'v'
) => {
const dates: { year: string; month: string; day: string }[] = [];
const fromDate = new Date(from * 1000);
const toDate = new Date(to * 1000);
for (let d = fromDate; d <= toDate; d.setDate(d.getDate() + 1)) {
dates.push({
year: d.getUTCFullYear().toString(),
month: (d.getUTCMonth() + 1).toString().padStart(2, '0'),
day: d.getUTCDate().toString().padStart(2, '0'),
});
}
return `WITH time_range AS (
SELECT
${from} AS from_ts,
${to} AS to_ts
),
valid_partitions AS (
SELECT * FROM (VALUES
${dates.map((d) => `('${d.year}', '${d.month}', '${d.day}')`).join(', ')}
) AS ${valid_partitions_label}(year, month, day)
)`;
};

51
src/index.ts
View File

@@ -50,8 +50,9 @@ import FEATURE_FLAGS from './utils/featureFlags';
import { getDLOBProviderFromOrderSubscriber } from './dlobProvider';
import { setGlobalDispatcher, Agent } from 'undici';
import { HermesClient } from '@pythnetwork/hermes-client';
import { COMMON_UI_UTILS } from '@drift/common';
import { COMMON_UI_UTILS, ENUM_UTILS } from '@drift/common';
import { AuctionParamArgs } from './utils/types';
import { TakerFillVsOracleBpsRedisResult } from './athena/repositories/fillQualityAnalytics';
setGlobalDispatcher(
new Agent({
@@ -947,6 +948,7 @@ const main = async (): Promise<void> => {
forceUpToSlippage,
maxLeverageSelected,
maxLeverageOrderSize,
version,
} = req.query;
// Validate required parameters
@@ -959,6 +961,31 @@ const main = async (): Promise<void> => {
return;
}
const apiVersion = version ? parseInt(version as string) : 1;
let redisFillQualityInfo: TakerFillVsOracleBpsRedisResult | undefined;
if (apiVersion === 2) {
const redisKey = `taker_fill_vs_oracle_bps:market:${marketIndex}`;
try {
const redisValue = await fetchFromRedis(
redisKey,
(responses) => responses[0] as any
);
if (redisValue) {
const parsed = JSON.parse(redisValue);
redisFillQualityInfo =
typeof parsed === 'string' ? JSON.parse(parsed) : parsed;
}
// Fall through to existing logic below
} catch (err) {
logger.error(
`Version 2: Error fetching redis stats for market ${marketIndex}:`,
err
);
// Fall through to existing logic below
}
}
// Parse and validate values
const parsedMarketIndex = parseInt(marketIndex as string);
if (isNaN(parsedMarketIndex)) {
@@ -1029,7 +1056,9 @@ const main = async (): Promise<void> => {
inputParams,
driftClient,
fetchFromRedis,
selectMostRecentBySlot
selectMostRecentBySlot,
redisFillQualityInfo,
apiVersion
);
if (!result.success) {
@@ -1043,6 +1072,24 @@ const main = async (): Promise<void> => {
result.data.marketOrderParams
);
// Log final auction prices for debugging
logger.info(
JSON.stringify({
event: 'auction_params_derived',
marketIndex: parsedMarketIndex,
direction: direction,
apiVersion,
finalAuctionParams: {
auctionStartPrice: auctionParams.auctionStartPrice?.toString(),
auctionEndPrice: auctionParams.auctionEndPrice?.toString(),
price: auctionParams.price?.toString(),
oraclePriceOffset: auctionParams.oraclePriceOffset?.toString(),
auctionDuration: auctionParams.auctionDuration?.toString(),
orderType: ENUM_UTILS.toStr(auctionParams.orderType),
},
})
);
const response = {
data: {
params: formatAuctionParamsForResponse(auctionParams),

229
src/publishers/dlobPublisher.ts
View File

@@ -43,6 +43,10 @@ import { handleHealthCheck } from '../core/middleware';
import { setGlobalDispatcher, Agent } from 'undici';
import { Metrics } from '../core/metricsV2';
import { OrderSubscriberFiltered } from '../dlob-subscriber/OrderSubscriberFiltered';
import {
FillQualityAnalyticsRepository,
TakerFillVsOracleBpsRedisResult,
} from '../athena/repositories/fillQualityAnalytics';
setGlobalDispatcher(
new Agent({
@@ -93,6 +97,10 @@ const tobStuckGauge = metricsV2.addGauge(
'tob_stuck_duration',
'Duration TOB has been stuck for each market'
);
const takerFillBpsFromOracleGauge = metricsV2.addGauge(
'taker_fill_bps_from_oracle',
'Taker fill BPS from oracle by market, side, and cohort'
);
metricsV2.finalizeObservables();
//@ts-ignore
@@ -109,7 +117,7 @@ const endpoint = process.env.ENDPOINT;
const grpcEndpoint = useGrpc
? process.env.GRPC_ENDPOINT ?? endpoint + `/${token}`
: '';
const grpcClient = process.env.GRPC_CLIENT ?? 'yellowstone'
const grpcClient = process.env.GRPC_CLIENT ?? 'yellowstone';
const wsEndpoint = process.env.WS_ENDPOINT;
const useOrderSubscriber =
@@ -122,6 +130,16 @@ const WS_FALLBACK_FETCH_INTERVAL = 60_000;
const KILLSWITCH_SLOT_DIFF_THRESHOLD =
parseInt(process.env.KILLSWITCH_SLOT_DIFF_THRESHOLD) || 200;
// Fill Quality Analytics configuration
const ENABLE_FILL_QUALITY_ANALYTICS =
process.env.ENABLE_FILL_QUALITY_ANALYTICS?.toLowerCase() === 'true';
const FILL_QUALITY_ANALYTICS_INTERVAL =
parseInt(process.env.FILL_QUALITY_ANALYTICS_INTERVAL) || 300_000; // 5 minutes default
const FILL_QUALITY_ANALYTICS_LOOKBACK_MS =
parseInt(process.env.FILL_QUALITY_ANALYTICS_LOOKBACK_MS) || 86_400_000; // 24 hours default
const FILL_QUALITY_ANALYTICS_SMOOTHING_MINUTES =
parseInt(process.env.FILL_QUALITY_ANALYTICS_SMOOTHING_MINUTES) || 60;
// TOB monitoring configuration - defaults to true if not set
const ENABLE_TOB_MONITORING =
!process.env.ENABLE_TOB_MONITORING ||
@@ -169,6 +187,22 @@ if (ENABLE_TOB_MONITORING) {
`TOB Monitoring Markets: ${TOB_MONITORING_ENABLED_PERP_MARKETS.join(', ')}`
);
}
logger.info(
`Fill Quality Analytics: ${
ENABLE_FILL_QUALITY_ANALYTICS ? 'enabled' : 'disabled'
}`
);
if (ENABLE_FILL_QUALITY_ANALYTICS) {
logger.info(
`Fill Quality Analytics Interval: ${FILL_QUALITY_ANALYTICS_INTERVAL}ms`
);
logger.info(
`Fill Quality Analytics Lookback: ${FILL_QUALITY_ANALYTICS_LOOKBACK_MS}ms`
);
logger.info(
`Fill Quality Analytics Smoothing: ${FILL_QUALITY_ANALYTICS_SMOOTHING_MINUTES} minutes`
);
}
let MARKET_SUBSCRIBERS: SubscriberLookup = {};
@@ -504,8 +538,7 @@ const main = async () => {
'grpc.keepalive_timeout_ms': 1_000,
'grpc.keepalive_permit_without_calls': 1,
},
client: grpcClient
client: grpcClient,
},
commitment: stateCommitment,
};
@@ -817,6 +850,196 @@ const main = async () => {
// Start TOB monitoring
setInterval(checkTobForStuckOrders, TOB_CHECK_INTERVAL);
// Track last known values for fill quality metrics (per market/side/cohort)
const lastFillQualityValues = new Map<string, number>();
// Helper function to update fill quality metric with null tracking
const updateFillQualityMetric = (
value: string | number | null | undefined,
marketIndex: string | number,
side: string,
cohort: string
) => {
const marketIndexStr = marketIndex.toString();
const key = `${marketIndexStr}:${side}:${cohort}`;
const isNull = value === null || value === undefined;
let valueToUse: number;
if (isNull) {
// Use last known value, or 0 if no previous value exists
valueToUse = lastFillQualityValues.get(key) || 0;
logger.debug(
`Null value for market ${marketIndexStr} ${side} ${cohort}, using last known value: ${valueToUse}`
);
} else {
// Use current value and update the last known value
valueToUse = Number(value);
lastFillQualityValues.set(key, valueToUse);
}
takerFillBpsFromOracleGauge.setLatestValue(valueToUse, {
marketIndex: marketIndexStr,
side,
cohort,
null: isNull ? 'true' : 'false',
});
};
// Fill Quality Analytics function - fetch and store in Redis
const fetchAndStoreFillQualityAnalytics = async () => {
if (!ENABLE_FILL_QUALITY_ANALYTICS) {
return;
}
try {
logger.info('Starting fill quality analytics fetch');
const startTime = Date.now();
const fillQualityRepo = FillQualityAnalyticsRepository();
const toMs = Date.now();
const fromMs = toMs - FILL_QUALITY_ANALYTICS_LOOKBACK_MS;
const results = await fillQualityRepo.getTakerFillVsOracleBps(
fromMs,
toMs,
9, // baseDecimals
FILL_QUALITY_ANALYTICS_SMOOTHING_MINUTES
);
logger.info(
`Fetched fill quality analytics for ${results.length} markets in ${
Date.now() - startTime
}ms`
);
// Store results in Redis - one key per market
const startRedisSet = Date.now();
for (const result of results) {
const redisKey = `taker_fill_vs_oracle_bps:market:${result.MarketIndex}`;
const redisValue = JSON.stringify({
marketIndex: result.MarketIndex,
takerBuyBpsFromOracle: {
all: result.TakerBuyBpsFromOracle_ALL,
'1e0': result.TakerBuyBpsFromOracle_1e0,
'1e3': result.TakerBuyBpsFromOracle_1e3,
'1e4': result.TakerBuyBpsFromOracle_1e4,
'1e5': result.TakerBuyBpsFromOracle_1e5,
'1e6': result.TakerBuyBpsFromOracle_1e6,
},
takerSellBpsFromOracle: {
all: result.TakerSellBpsFromOracle_ALL,
'1e0': result.TakerSellBpsFromOracle_1e0,
'1e3': result.TakerSellBpsFromOracle_1e3,
'1e4': result.TakerSellBpsFromOracle_1e4,
'1e5': result.TakerSellBpsFromOracle_1e5,
'1e6': result.TakerSellBpsFromOracle_1e6,
},
updatedAtTs: Date.now(),
} as TakerFillVsOracleBpsRedisResult);
await redisClient.set(redisKey, redisValue);
updateFillQualityMetric(
result.TakerBuyBpsFromOracle_ALL,
result.MarketIndex,
'buy',
'all'
);
updateFillQualityMetric(
result.TakerBuyBpsFromOracle_1e0,
result.MarketIndex,
'buy',
'1e0'
);
updateFillQualityMetric(
result.TakerBuyBpsFromOracle_1e3,
result.MarketIndex,
'buy',
'1e3'
);
updateFillQualityMetric(
result.TakerBuyBpsFromOracle_1e4,
result.MarketIndex,
'buy',
'1e4'
);
updateFillQualityMetric(
result.TakerBuyBpsFromOracle_1e5,
result.MarketIndex,
'buy',
'1e5'
);
updateFillQualityMetric(
result.TakerBuyBpsFromOracle_1e6,
result.MarketIndex,
'buy',
'1e6'
);
updateFillQualityMetric(
result.TakerSellBpsFromOracle_ALL,
result.MarketIndex,
'sell',
'all'
);
updateFillQualityMetric(
result.TakerSellBpsFromOracle_1e0,
result.MarketIndex,
'sell',
'1e0'
);
updateFillQualityMetric(
result.TakerSellBpsFromOracle_1e3,
result.MarketIndex,
'sell',
'1e3'
);
updateFillQualityMetric(
result.TakerSellBpsFromOracle_1e4,
result.MarketIndex,
'sell',
'1e4'
);
updateFillQualityMetric(
result.TakerSellBpsFromOracle_1e5,
result.MarketIndex,
'sell',
'1e5'
);
updateFillQualityMetric(
result.TakerSellBpsFromOracle_1e6,
result.MarketIndex,
'sell',
'1e6'
);
logger.debug(
`Stored fill quality analytics for market ${result.MarketIndex}`
);
}
logger.info(
`Successfully stored fill quality analytics for ${
results.length
} markets in Redis in ${Date.now() - startRedisSet}ms`
);
} catch (error) {
logger.error('Error fetching/storing fill quality analytics:', error);
}
};
// Run fill quality analytics fetch immediately on startup, then on interval
if (ENABLE_FILL_QUALITY_ANALYTICS) {
fetchAndStoreFillQualityAnalytics().catch((error) => {
logger.error('Initial fill quality analytics fetch failed:', error);
});
setInterval(
fetchAndStoreFillQualityAnalytics,
FILL_QUALITY_ANALYTICS_INTERVAL
);
logger.info(
`Fill quality analytics will run every ${FILL_QUALITY_ANALYTICS_INTERVAL}ms`
);
}
const handleStartup = async (_req, res, _next) => {
if (driftClient.isSubscribed && dlobProvider.size() > 0) {
res.writeHead(200);

154
src/utils/utils.ts
View File

@@ -34,6 +34,7 @@ import { wsMarketArgs } from 'src/dlob-subscriber/DLOBSubscriberIO';
import { DEFAULT_AUCTION_PARAMS, MID_MAJOR_MARKETS } from './constants';
import { AuctionParamArgs } from './types';
import { COMMON_MATH, ENUM_UTILS } from '@drift/common';
import { TakerFillVsOracleBpsRedisResult } from '../athena/repositories/fillQualityAnalytics';
export const GROUPING_OPTIONS = [1, 10, 100, 500, 1000];
export const GROUPING_DEPENDENCIES = {
@@ -312,10 +313,17 @@ export function publishGroupings(
asks: aggregatedAsks,
});
if(['SOL-PERP', 'BTC-PERP', 'ETH-PERP'].includes(l2Formatted_grouped20.marketName) && aggregatedBids.length !== 20 || aggregatedAsks.length !== 20) {
logger.error(`Error aggregating dlob levels: group=${group}, bids=${fullAggregatedBids.length}, asks=${fullAggregatedAsks.length}`)
logger.error(`Response: ${JSON.stringify(l2Formatted_grouped20)}`)
if (
(['SOL-PERP', 'BTC-PERP', 'ETH-PERP'].includes(
l2Formatted_grouped20.marketName
) &&
aggregatedBids.length !== 20) ||
aggregatedAsks.length !== 20
) {
logger.error(
`Error aggregating dlob levels: group=${group}, bids=${fullAggregatedBids.length}, asks=${fullAggregatedAsks.length}`
);
logger.error(`Response: ${JSON.stringify(l2Formatted_grouped20)}`);
}
redisClient.publish(
@@ -911,6 +919,8 @@ export const getEstimatedPrices = async (
* @param driftClient - DriftClient instance (optional, for price calculation)
* @param fetchFromRedis - Redis fetch function (optional, for price calculation)
* @param selectMostRecentBySlot - Slot selection function (optional, for price calculation)
* @param fillQualityInfo - Fill quality analytics data (version 2 only)
* @param apiVersion - API version (1 or 2)
* @returns Object formatted for deriveMarketOrderParams function or error response
*/
export const mapToMarketOrderParams = async (
@@ -920,7 +930,9 @@ export const mapToMarketOrderParams = async (
key: string,
selectionCriteria: (responses: any) => any
) => Promise<any>,
selectMostRecentBySlot?: (responses: any[]) => any
selectMostRecentBySlot?: (responses: any[]) => any,
fillQualityInfo?: TakerFillVsOracleBpsRedisResult,
apiVersion: number = 1
): Promise<{
success: boolean;
data?: {
@@ -960,6 +972,15 @@ export const mapToMarketOrderParams = async (
let estimatedPrices;
let processedSlippageTolerance = params.slippageTolerance;
// Track debug info for logging
const debugInfo = {
originalOraclePrice: null as string | null,
adjustedOraclePrice: null as string | null,
isCrossed: false,
fillQualityBps: null as number | null,
appliedV2Adjustment: false,
};
if (driftClient && fetchFromRedis && selectMostRecentBySlot) {
// Get L2 orderbook data using the utility function
const redisL2 = await fetchL2FromRedis(
@@ -980,6 +1001,73 @@ export const mapToMarketOrderParams = async (
redisL2
);
// Store original oracle for debugging
debugInfo.originalOraclePrice = estimatedPrices.oraclePrice.toString();
// VERSION 2: Adjust oracle price based on fill quality when orderbook is crossed
if (apiVersion === 2 && fillQualityInfo && redisL2) {
try {
// Convert raw L2 to formatted L2 for cross detection
const l2Formatted = convertRawL2ToBN(redisL2);
const isSpot = isVariant(marketType, 'spot');
const oracleData = isSpot
? driftClient.getOracleDataForSpotMarket(params.marketIndex)
: driftClient.getOracleDataForPerpMarket(params.marketIndex);
const oraclePrice = oracleData.price ?? ZERO;
// Detect if orderbook is crossed
const spreadInfo = COMMON_MATH.calculateSpreadBidAskMark(
l2Formatted,
oraclePrice
);
const isCrossed =
spreadInfo.bestBidPrice &&
spreadInfo.bestAskPrice &&
spreadInfo.bestBidPrice.gte(spreadInfo.bestAskPrice);
debugInfo.isCrossed = isCrossed;
if (isCrossed) {
// Get fill quality metric based on direction
const fillQualityBpsStr =
direction === PositionDirection.LONG
? fillQualityInfo.takerBuyBpsFromOracle?.all
: fillQualityInfo.takerSellBpsFromOracle?.all;
if (
fillQualityBpsStr &&
fillQualityBpsStr !== 'null' &&
fillQualityBpsStr !== null
) {
const fillQualityBps = Math.round(
parseFloat(fillQualityBpsStr) * 100
);
debugInfo.fillQualityBps = fillQualityBps;
if (!isNaN(fillQualityBps)) {
const adjustment = oraclePrice
.muln(fillQualityBps)
.divn(10000 * 100); // adjustmentFactor is in bps
const adjustedOraclePrice = oraclePrice.add(adjustment);
// Update the oracle price in estimatedPrices
estimatedPrices.oraclePrice = adjustedOraclePrice;
debugInfo.adjustedOraclePrice = adjustedOraclePrice.toString();
debugInfo.appliedV2Adjustment = true;
}
}
}
} catch (error) {
logger.warn(
'Version 2: Failed to apply fill quality adjustment, using standard oracle:',
error
);
// Fall through to use unadjusted oracle
}
}
// Handle dynamic slippage tolerance calculation if needed
if (params.slippageTolerance === undefined) {
// Convert raw L2 to formatted L2 for slippage calculation
@@ -1028,6 +1116,62 @@ export const mapToMarketOrderParams = async (
baseAmount = amount.mul(BASE_PRECISION).div(estimatedPrices.entryPrice);
}
// Comprehensive debug logging
logger.info(
JSON.stringify({
event: 'auction_params_calculated',
requestParams: {
marketIndex: params.marketIndex,
marketType: params.marketType,
direction: direction === PositionDirection.LONG ? 'long' : 'short',
amount: params.amount,
assetType: params.assetType,
slippageTolerance: params.slippageTolerance,
apiVersion,
},
priceDiscovery: {
originalOraclePrice: debugInfo.originalOraclePrice,
finalOraclePrice: estimatedPrices.oraclePrice.toString(),
bestPrice: estimatedPrices.bestPrice.toString(),
entryPrice: estimatedPrices.entryPrice.toString(),
worstPrice: estimatedPrices.worstPrice.toString(),
markPrice: estimatedPrices.markPrice.toString(),
priceImpactBps: estimatedPrices.priceImpact.toString(),
},
v2CrossDetection:
apiVersion === 2
? {
isCrossed: debugInfo.isCrossed,
fillQualityBps: debugInfo.fillQualityBps,
adjustedOraclePrice: debugInfo.adjustedOraclePrice,
appliedAdjustment: debugInfo.appliedV2Adjustment,
fillQualityData: fillQualityInfo
? {
takerBuyBpsAll: fillQualityInfo.takerBuyBpsFromOracle?.all,
takerSellBpsAll:
fillQualityInfo.takerSellBpsFromOracle?.all,
}
: null,
}
: undefined,
auctionConfig: {
duration: params.auctionDuration,
startPriceOffset: params.auctionStartPriceOffset,
startPriceOffsetFrom: params.auctionStartPriceOffsetFrom,
endPriceOffset: params.auctionEndPriceOffset,
endPriceOffsetFrom: params.auctionEndPriceOffsetFrom,
slippageTolerance: processedSlippageTolerance,
isOracleOrder: params.isOracleOrder,
reduceOnly: params.reduceOnly ?? false,
allowInfSlippage: params.allowInfSlippage ?? false,
},
calculatedValues: {
baseAmount: baseAmount.toString(),
slippageToleranceFinal: processedSlippageTolerance,
},
})
);
return {
success: true,
data: {

761
yarn.lock
View File

File diff suppressed because it is too large Load Diff