Merge pull request #537 from drift-labs/wphan/auc-params-v2

Wphan/auc params v2

package.json

@@ -5,6 +5,7 @@
 	"main": "lib/index.js",
 	"license": "Apache-2.0",
 	"dependencies": {
+		"@aws-sdk/client-athena": "^3.830.0",
 		"@aws-sdk/client-kinesis": "^3.830.0",
 		"@aws-sdk/util-retry": "^3.374.0",
 		"@coral-xyz/anchor": "^0.29.0",

src/athena/client.ts

@@ -0,0 +1,237 @@
+import {
+	AthenaClient,
+	ColumnInfo,
+	GetQueryExecutionCommand,
+	GetQueryResultsCommand,
+	GetQueryResultsCommandOutput,
+	QueryExecutionState,
+	Row,
+	StartQueryExecutionCommand,
+	StartQueryExecutionCommandInput,
+} from '@aws-sdk/client-athena';
+import { ConfiguredRetryStrategy } from '@aws-sdk/util-retry';
+import Bottleneck from 'bottleneck';
+import { logger } from '../utils/logger';
+
+const QUERY_TIMEOUT = 300000;
+const POLL_INTERVAL = 1000;
+
+const DEFAULT_ATHENA_DATABASE = 'staging-archive';
+const DEFAULT_ATHENA_OUTPUT_BUCKET = 'staging-data-ingestion-bucket';
+
+const limiter = new Bottleneck({
+	maxConcurrent: 5,
+	minTime: 100,
+});
+
+type QueryResult = Record<string, string | null>;
+
+const parseRow = (row: Row, columnInfo: ColumnInfo[]): QueryResult => {
+	const rowData = row.Data;
+	if (!rowData) {
+		return {};
+	}
+
+	return columnInfo.reduce((acc: QueryResult, column, index) => {
+		if (column.Name) {
+			acc[column.Name] = rowData[index]?.VarCharValue ?? null;
+		}
+		return acc;
+	}, {});
+};
+
+const athena = new AthenaClient({
+	region: process.env.AWS_REGION,
+	retryStrategy: new ConfiguredRetryStrategy(
+		3,
+		(attempt: number) => 100 + attempt * 1000
+	),
+});
+
+export const Athena = ({
+	overrideDatabaseName,
+	overrideBucketName,
+}: { overrideDatabaseName?: string; overrideBucketName?: string } = {}) => {
+	const database =
+		overrideDatabaseName ??
+		process.env.ATHENA_DATABASE ??
+		DEFAULT_ATHENA_DATABASE;
+	const outputLocation = `s3://${
+		overrideBucketName ??
+		process.env.ATHENA_OUTPUT_BUCKET ??
+		DEFAULT_ATHENA_OUTPUT_BUCKET
+	}/athena`;
+
+	const startQuery = async (query: string, params?: Record<string, string>) => {
+		const executionParams: StartQueryExecutionCommandInput = {
+			QueryString: query,
+			QueryExecutionContext: {
+				Database: database,
+			},
+			ResultConfiguration: {
+				OutputLocation: outputLocation,
+			},
+			ResultReuseConfiguration: {
+				ResultReuseByAgeConfiguration: {
+					Enabled: false,
+				},
+			},
+		};
+
+		if (params) {
+			executionParams.ExecutionParameters = Object.values(params);
+		}
+
+		const { QueryExecutionId } = await athena.send(
+			new StartQueryExecutionCommand(executionParams)
+		);
+
+		if (!QueryExecutionId) {
+			throw new Error('Failed to start query execution');
+		}
+
+		return QueryExecutionId;
+	};
+
+	const getQueryExecution = async (queryExecutionId: string) => {
+		const { QueryExecution } = await athena.send(
+			new GetQueryExecutionCommand({
+				QueryExecutionId: queryExecutionId,
+			})
+		);
+
+		return QueryExecution;
+	};
+
+	const waitForQueryCompletion = async (
+		queryExecutionId: string,
+		timeout: number = QUERY_TIMEOUT
+	): Promise<void> => {
+		const startTime = Date.now();
+
+		while (Date.now() - startTime < timeout) {
+			const execution = await getQueryExecution(queryExecutionId);
+			logger.info(
+				`Total bytes scanned: ${
+					(execution?.Statistics?.DataScannedInBytes ?? 0) / 1024
+				}kb`
+			);
+			const state = execution?.Status?.State;
+
+			switch (state) {
+				case QueryExecutionState.SUCCEEDED:
+					return;
+				case QueryExecutionState.FAILED:
+					throw new Error(
+						`Query failed: ${execution?.Status?.StateChangeReason}`
+					);
+				case QueryExecutionState.CANCELLED:
+					throw new Error('Query was cancelled');
+				default:
+					await new Promise((resolve) => setTimeout(resolve, POLL_INTERVAL));
+			}
+		}
+
+		throw new Error(`Query timeout after ${timeout}ms`);
+	};
+
+	const getQueryResults = async (
+		queryExecutionId: string,
+		nextToken?: string
+	): Promise<GetQueryResultsCommandOutput> => {
+		return athena.send(
+			new GetQueryResultsCommand({
+				QueryExecutionId: queryExecutionId,
+				NextToken: nextToken,
+			})
+		);
+	};
+
+	const getAllQueryResults = async (
+		queryExecutionId: string
+	): Promise<QueryResult[]> => {
+		let nextToken: string | undefined;
+		const allResults: QueryResult[] = [];
+
+		do {
+			const results = await getQueryResults(queryExecutionId, nextToken);
+			if (
+				results.ResultSet?.Rows &&
+				results.ResultSet.ResultSetMetadata?.ColumnInfo
+			) {
+				const columnInfo = results.ResultSet.ResultSetMetadata.ColumnInfo;
+				const rows = nextToken
+					? results.ResultSet.Rows
+					: results.ResultSet.Rows.slice(1);
+				const parsedRows = rows.map((row) => parseRow(row, columnInfo));
+				allResults.push(...parsedRows);
+			}
+
+			nextToken = results.NextToken;
+		} while (nextToken);
+
+		return allResults;
+	};
+
+	const query = async (
+		queryString: string,
+		params?: Record<string, string>
+	): Promise<QueryResult[]> => {
+		try {
+			const queryExecutionId = await startQuery(queryString, params);
+			await waitForQueryCompletion(queryExecutionId);
+			return getAllQueryResults(queryExecutionId);
+		} catch (error) {
+			const { message } = error as Error;
+			logger.error(`Error executing Athena query: ${message}`);
+			throw error;
+		}
+	};
+
+	const batchQuery = async ({
+		queries,
+	}: {
+		queries: { query: string; params?: Record<string, string> }[];
+	}): Promise<{
+		results: QueryResult[][];
+		errors: Error[];
+	}> => {
+		const results: QueryResult[][] = [];
+		const errors: Error[] = [];
+
+		await Promise.all(
+			queries.map(async ({ query: queryString, params }, index) => {
+				try {
+					const result = await limiter.schedule(() =>
+						query(queryString, params)
+					);
+					results[index] = result;
+				} catch (error) {
+					logger.error(
+						`Error in batch query ${index}: ${queryString}, Error: ${error}`
+					);
+					errors[index] = error as Error;
+				}
+			})
+		);
+
+		if (errors.length > 0) {
+			logger.warn(`${errors.length} queries failed in batch execution`);
+		}
+
+		return {
+			results,
+			errors,
+		};
+	};
+
+	return {
+		query,
+		batchQuery,
+		startQuery,
+		getQueryExecution,
+		getQueryResults,
+		getAllQueryResults,
+		waitForQueryCompletion,
+	};
+};

src/athena/index.ts

@@ -0,0 +1,3 @@
+export * from './client';
+export * from './utils';
+export * from './repositories/fillQualityAnalytics';

src/athena/repositories/fillQualityAnalytics.ts

@@ -0,0 +1,298 @@
+import { Athena } from '../client';
+
+export interface TakerFillVsOracleBpsResult {
+	MarketIndex: string;
+	TakerBuyBpsFromOracle_ALL: string | null;
+	TakerSellBpsFromOracle_ALL: string | null;
+	TakerBuyBpsFromOracle_1e0: string | null;
+	TakerBuyBpsFromOracle_1e3: string | null;
+	TakerBuyBpsFromOracle_1e4: string | null;
+	TakerBuyBpsFromOracle_1e5: string | null;
+	TakerBuyBpsFromOracle_1e6: string | null;
+	TakerSellBpsFromOracle_1e0: string | null;
+	TakerSellBpsFromOracle_1e3: string | null;
+	TakerSellBpsFromOracle_1e4: string | null;
+	TakerSellBpsFromOracle_1e5: string | null;
+	TakerSellBpsFromOracle_1e6: string | null;
+	Zero: string;
+}
+
+export type TakerFillVsOracleBpsRedisResult = {
+	marketIndex: string;
+	takerBuyBpsFromOracle: {
+		all: string | null;
+		'1e0': string | null;
+		'1e3': string | null;
+		'1e4': string | null;
+		'1e5': string | null;
+		'1e6': string | null;
+	};
+	takerSellBpsFromOracle: {
+		all: string | null;
+		'1e0': string | null;
+		'1e3': string | null;
+		'1e4': string | null;
+		'1e5': string | null;
+		'1e6': string | null;
+	};
+	updatedAtTs: number;
+};
+
+export const FillQualityAnalyticsRepository = () => {
+	const { query } = Athena();
+
+	/**
+	 * Get taker fill vs oracle basis points, bucketed by order notional cohorts at place-time size.
+	 * Returns a rolling average per cohort/direction, partitioned by market.
+	 * This query returns data for ALL perp markets in a single result set.
+	 *
+	 * @param from - Unix timestamp in milliseconds
+	 * @param to - Unix timestamp in milliseconds
+	 * @param baseDecimals - Base decimals for the market (default: 9)
+	 * @param smoothingMinutes - Minutes for rolling average (default: 60)
+	 * @returns Array of time series data with taker fill vs oracle bps by cohort and market
+	 */
+	const getTakerFillVsOracleBps = async (
+		fromMs: number,
+		toMs: number,
+		baseDecimals: number = 9,
+		smoothingMinutes: number = 60
+	): Promise<TakerFillVsOracleBpsResult[]> => {
+		// Taker fill vs Oracle bps, bucketed by order notional cohorts at place-time size
+		// Final output: latest non-null rolling average per cohort/direction, PARTITIONED BY market
+		const queryString = `
+WITH
+from_dt AS (
+  SELECT
+    date_format(from_unixtime(CAST(${fromMs}/1000 AS BIGINT)) AT TIME ZONE 'UTC','%Y') AS yf,
+    date_format(from_unixtime(CAST(${fromMs}/1000 AS BIGINT)) AT TIME ZONE 'UTC','%m') AS mf,
+    date_format(from_unixtime(CAST(${fromMs}/1000 AS BIGINT)) AT TIME ZONE 'UTC','%d') AS df
+),
+to_dt AS (
+  SELECT
+    date_format(from_unixtime(CAST(${toMs}/1000 AS BIGINT)) AT TIME ZONE 'UTC','%Y') AS yt,
+    date_format(from_unixtime(CAST(${toMs}/1000 AS BIGINT)) AT TIME ZONE 'UTC','%m') AS mt,
+    date_format(from_unixtime(CAST(${toMs}/1000 AS BIGINT)) AT TIME ZONE 'UTC','%d') AS dt
+),
+
+-- Dedup to the earliest OrderRecord per (txsig,user,orderid,marketindex) to reflect "place" state
+orders_dedup AS (
+  SELECT *
+  FROM (
+    SELECT
+      CAST(orx.ts AS BIGINT)                                        AS order_ts_epoch,
+      orx.txsig                                                     AS txsig,
+      orx.user                                                      AS user,
+      orx."order".orderid                                           AS orderid,
+      orx."order".marketindex                                       AS marketindex,
+      CAST(orx."order".baseassetamount AS DOUBLE)                   AS base_asset_amount_raw,
+      ROW_NUMBER() OVER (
+        PARTITION BY orx.txsig, orx.user, orx."order".orderid, orx."order".marketindex
+        ORDER BY CAST(orx.ts AS BIGINT) ASC
+      ) AS rn
+    FROM eventtype_orderrecord orx
+    CROSS JOIN from_dt f
+    CROSS JOIN to_dt t
+    WHERE
+      orx."order".markettype = 'perp'
+      -- UTC-safe pruning on VARCHAR partitions
+      AND orx.year BETWEEN f.yf AND t.yt
+      AND (
+            orx.year >  f.yf
+         OR (orx.year = f.yf AND orx.month >  f.mf)
+         OR (orx.year = f.yf AND orx.month = f.mf AND orx.day >= f.df)
+      )
+      AND (
+            orx.year <  t.yt
+         OR (orx.year = t.yt AND orx.month <  t.mt)
+         OR (orx.year = t.yt AND orx.month = t.mt AND orx.day <= t.dt)
+      )
+      -- absolute time guardrails (ts is epoch seconds in table)
+      AND CAST(orx.ts AS BIGINT) BETWEEN CAST(${fromMs}/1000 AS BIGINT) AND CAST(${toMs}/1000 AS BIGINT)
+  ) t1
+  WHERE rn = 1
+),
+
+-- Trade fills (taker perspective) within the window (all markets)
+trades AS (
+  SELECT
+    CAST(tr.ts AS BIGINT)                               AS ts_epoch,
+    tr.txsig                                            AS txsig,
+    tr.taker                                            AS user,
+    tr.takerorderid                                     AS orderid,
+    tr.marketindex                                      AS marketindex,
+    LOWER(tr.takerorderdirection)                       AS dir,
+    CAST(tr.quoteassetamountfilled AS DOUBLE)           AS q_filled,
+    CAST(tr.baseassetamountfilled  AS DOUBLE)           AS b_filled,
+    CAST(tr.oracleprice            AS DOUBLE)           AS oracle_raw
+  FROM eventtype_traderecord tr
+  CROSS JOIN from_dt f
+  CROSS JOIN to_dt t
+  WHERE
+    LOWER(tr.action) = 'fill'
+    AND tr.markettype = 'perp'
+    AND LOWER(tr.takerorderdirection) IN ('long','short')
+
+    -- UTC-safe pruning on VARCHAR partitions
+    AND tr.year BETWEEN f.yf AND t.yt
+    AND (
+          tr.year >  f.yf
+       OR (tr.year = f.yf AND tr.month >  f.mf)
+       OR (tr.year = f.yf AND tr.month = f.mf AND tr.day >= f.df)
+    )
+    AND (
+          tr.year <  t.yt
+       OR (tr.year = t.yt AND tr.month <  t.mt)
+       OR (tr.year = t.yt AND tr.month = t.mt AND tr.day <= t.dt)
+    )
+
+    -- absolute time guardrails (ts is epoch seconds in table)
+    AND CAST(tr.ts AS BIGINT) BETWEEN CAST(${fromMs}/1000 AS BIGINT) AND CAST(${toMs}/1000 AS BIGINT)
+),
+
+-- Join fills to the order's initial size; compute notional cohort and taker bps from oracle
+joined AS (
+  SELECT
+    from_unixtime(tr.ts_epoch) AS Time,
+    tr.marketindex             AS MarketIndex,
+    tr.dir                     AS dir,
+    (
+      (
+        (tr.q_filled / NULLIF(tr.b_filled, 0))
+        * pow(10.0, (${baseDecimals} - 6))
+      )
+      / (tr.oracle_raw / 1e6) - 1.0
+    ) * 10000.0 AS taker_bps_from_oracle,
+    ( (od.base_asset_amount_raw * 1e-9) * (tr.oracle_raw * 1e-6) ) AS order_value
+  FROM trades tr
+  JOIN orders_dedup od
+    ON  od.txsig       = tr.txsig
+    AND od.user        = tr.user
+    AND od.orderid     = tr.orderid
+    AND od.marketindex = tr.marketindex
+),
+
+-- Bucket into cohorts
+bucketed AS (
+  SELECT
+    Time,
+    MarketIndex,
+    dir,
+    taker_bps_from_oracle,
+    CASE
+      WHEN order_value > 0      AND order_value < 1000       THEN '1e0'
+      WHEN order_value >= 1000  AND order_value < 10000      THEN '1e3'
+      WHEN order_value >= 10000 AND order_value < 100000     THEN '1e4'
+      WHEN order_value >= 100000 AND order_value < 1000000   THEN '1e5'
+      WHEN order_value >= 1000000                            THEN '1e6'
+      ELSE 'other'
+    END AS cohort
+  FROM joined
+),
+
+-- Compute rolling averages
+rolling_avgs AS (
+  SELECT
+    Time,
+    MarketIndex,
+
+    -- Non-cohort series (ALL fills regardless of cohort)
+    AVG(CASE WHEN dir = 'long'  THEN taker_bps_from_oracle END)
+      OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
+      AS "TakerBuyBpsFromOracle_ALL",
+
+    AVG(CASE WHEN dir = 'short' THEN taker_bps_from_oracle END)
+      OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
+      AS "TakerSellBpsFromOracle_ALL",
+
+    -- rolling average by cohort/direction (NULLs ignored by AVG)
+    AVG(CASE WHEN dir = 'long'  AND cohort = '1e0' THEN taker_bps_from_oracle END)
+      OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
+      AS "TakerBuyBpsFromOracle_1e0",
+
+    AVG(CASE WHEN dir = 'long'  AND cohort = '1e3' THEN taker_bps_from_oracle END)
+      OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
+      AS "TakerBuyBpsFromOracle_1e3",
+
+    AVG(CASE WHEN dir = 'long'  AND cohort = '1e4' THEN taker_bps_from_oracle END)
+      OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
+      AS "TakerBuyBpsFromOracle_1e4",
+
+    AVG(CASE WHEN dir = 'long'  AND cohort = '1e5' THEN taker_bps_from_oracle END)
+      OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
+      AS "TakerBuyBpsFromOracle_1e5",
+
+    AVG(CASE WHEN dir = 'long'  AND cohort = '1e6' THEN taker_bps_from_oracle END)
+      OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
+      AS "TakerBuyBpsFromOracle_1e6",
+
+    AVG(CASE WHEN dir = 'short' AND cohort = '1e0' THEN taker_bps_from_oracle END)
+      OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
+      AS "TakerSellBpsFromOracle_1e0",
+
+    AVG(CASE WHEN dir = 'short' AND cohort = '1e3' THEN taker_bps_from_oracle END)
+      OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
+      AS "TakerSellBpsFromOracle_1e3",
+
+    AVG(CASE WHEN dir = 'short' AND cohort = '1e4' THEN taker_bps_from_oracle END)
+      OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
+      AS "TakerSellBpsFromOracle_1e4",
+
+    AVG(CASE WHEN dir = 'short' AND cohort = '1e5' THEN taker_bps_from_oracle END)
+      OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
+      AS "TakerSellBpsFromOracle_1e5",
+
+    AVG(CASE WHEN dir = 'short' AND cohort = '1e6' THEN taker_bps_from_oracle END)
+      OVER (PARTITION BY MarketIndex ORDER BY Time RANGE BETWEEN INTERVAL '${smoothingMinutes}' MINUTE PRECEDING AND CURRENT ROW)
+      AS "TakerSellBpsFromOracle_1e6",
+    
+    ROW_NUMBER() OVER (PARTITION BY MarketIndex ORDER BY Time DESC) AS rn
+  FROM bucketed
+  WHERE taker_bps_from_oracle IS NOT NULL
+)
+
+-- Select only the latest row per market
+SELECT
+  Time,
+  MarketIndex,
+  "TakerBuyBpsFromOracle_ALL",
+  "TakerSellBpsFromOracle_ALL",
+  "TakerBuyBpsFromOracle_1e0",
+  "TakerBuyBpsFromOracle_1e3",
+  "TakerBuyBpsFromOracle_1e4",
+  "TakerBuyBpsFromOracle_1e5",
+  "TakerBuyBpsFromOracle_1e6",
+  "TakerSellBpsFromOracle_1e0",
+  "TakerSellBpsFromOracle_1e3",
+  "TakerSellBpsFromOracle_1e4",
+  "TakerSellBpsFromOracle_1e5",
+  "TakerSellBpsFromOracle_1e6"
+FROM rolling_avgs
+WHERE rn = 1
+ORDER BY MarketIndex;
+		`;
+
+		const results = await query(queryString);
+
+		return results.map((result) => ({
+			MarketIndex: result.MarketIndex || '',
+			TakerBuyBpsFromOracle_ALL: result.TakerBuyBpsFromOracle_ALL,
+			TakerSellBpsFromOracle_ALL: result.TakerSellBpsFromOracle_ALL,
+			TakerBuyBpsFromOracle_1e0: result.TakerBuyBpsFromOracle_1e0,
+			TakerBuyBpsFromOracle_1e3: result.TakerBuyBpsFromOracle_1e3,
+			TakerBuyBpsFromOracle_1e4: result.TakerBuyBpsFromOracle_1e4,
+			TakerBuyBpsFromOracle_1e5: result.TakerBuyBpsFromOracle_1e5,
+			TakerBuyBpsFromOracle_1e6: result.TakerBuyBpsFromOracle_1e6,
+			TakerSellBpsFromOracle_1e0: result.TakerSellBpsFromOracle_1e0,
+			TakerSellBpsFromOracle_1e3: result.TakerSellBpsFromOracle_1e3,
+			TakerSellBpsFromOracle_1e4: result.TakerSellBpsFromOracle_1e4,
+			TakerSellBpsFromOracle_1e5: result.TakerSellBpsFromOracle_1e5,
+			TakerSellBpsFromOracle_1e6: result.TakerSellBpsFromOracle_1e6,
+			Zero: result.Zero || '0',
+		}));
+	};
+
+	return {
+		getTakerFillVsOracleBps,
+	};
+};

src/athena/utils.ts

@@ -0,0 +1,39 @@
+export const getTimePartition = (from: number, to: number) => {
+	return `WITH time_range AS (
+        SELECT 
+            ${from} as from_ts,
+            ${to} as to_ts,
+            DATE_FORMAT(from_unixtime(${from}), '%Y%m%d') as from_date,
+            DATE_FORMAT(from_unixtime(${to}), '%Y%m%d') as to_date
+        )`;
+};
+
+export const getTimeRangeAndPartitions = (
+	from: number,
+	to: number,
+	valid_partitions_label = 'v'
+) => {
+	const dates: { year: string; month: string; day: string }[] = [];
+	const fromDate = new Date(from * 1000);
+	const toDate = new Date(to * 1000);
+
+	for (let d = fromDate; d <= toDate; d.setDate(d.getDate() + 1)) {
+		dates.push({
+			year: d.getUTCFullYear().toString(),
+			month: (d.getUTCMonth() + 1).toString().padStart(2, '0'),
+			day: d.getUTCDate().toString().padStart(2, '0'),
+		});
+	}
+
+	return `WITH time_range AS (
+		SELECT 
+			${from} AS from_ts,
+			${to} AS to_ts
+	),
+
+	valid_partitions AS (
+		SELECT * FROM (VALUES
+			${dates.map((d) => `('${d.year}', '${d.month}', '${d.day}')`).join(', ')}
+		) AS ${valid_partitions_label}(year, month, day)
+	)`;
+};

src/index.ts

@@ -50,8 +50,9 @@ import FEATURE_FLAGS from './utils/featureFlags';
 import { getDLOBProviderFromOrderSubscriber } from './dlobProvider';
 import { setGlobalDispatcher, Agent } from 'undici';
 import { HermesClient } from '@pythnetwork/hermes-client';
-import { COMMON_UI_UTILS } from '@drift/common';
+import { COMMON_UI_UTILS, ENUM_UTILS } from '@drift/common';
 import { AuctionParamArgs } from './utils/types';
+import { TakerFillVsOracleBpsRedisResult } from './athena/repositories/fillQualityAnalytics';
 
 setGlobalDispatcher(
 	new Agent({
@@ -947,6 +948,7 @@ const main = async (): Promise<void> => {
 				forceUpToSlippage,
 				maxLeverageSelected,
 				maxLeverageOrderSize,
+				version,
 			} = req.query;
 
 			// Validate required parameters
@@ -959,6 +961,31 @@ const main = async (): Promise<void> => {
 				return;
 			}
 
+			const apiVersion = version ? parseInt(version as string) : 1;
+
+			let redisFillQualityInfo: TakerFillVsOracleBpsRedisResult | undefined;
+			if (apiVersion === 2) {
+				const redisKey = `taker_fill_vs_oracle_bps:market:${marketIndex}`;
+				try {
+					const redisValue = await fetchFromRedis(
+						redisKey,
+						(responses) => responses[0] as any
+					);
+					if (redisValue) {
+						const parsed = JSON.parse(redisValue);
+						redisFillQualityInfo =
+							typeof parsed === 'string' ? JSON.parse(parsed) : parsed;
+					}
+					// Fall through to existing logic below
+				} catch (err) {
+					logger.error(
+						`Version 2: Error fetching redis stats for market ${marketIndex}:`,
+						err
+					);
+					// Fall through to existing logic below
+				}
+			}
+
 			// Parse and validate values
 			const parsedMarketIndex = parseInt(marketIndex as string);
 			if (isNaN(parsedMarketIndex)) {
@@ -1029,7 +1056,9 @@ const main = async (): Promise<void> => {
 				inputParams,
 				driftClient,
 				fetchFromRedis,
-				selectMostRecentBySlot
+				selectMostRecentBySlot,
+				redisFillQualityInfo,
+				apiVersion
 			);
 
 			if (!result.success) {
@@ -1043,6 +1072,24 @@ const main = async (): Promise<void> => {
 				result.data.marketOrderParams
 			);
 
+			// Log final auction prices for debugging
+			logger.info(
+				JSON.stringify({
+					event: 'auction_params_derived',
+					marketIndex: parsedMarketIndex,
+					direction: direction,
+					apiVersion,
+					finalAuctionParams: {
+						auctionStartPrice: auctionParams.auctionStartPrice?.toString(),
+						auctionEndPrice: auctionParams.auctionEndPrice?.toString(),
+						price: auctionParams.price?.toString(),
+						oraclePriceOffset: auctionParams.oraclePriceOffset?.toString(),
+						auctionDuration: auctionParams.auctionDuration?.toString(),
+						orderType: ENUM_UTILS.toStr(auctionParams.orderType),
+					},
+				})
+			);
+
 			const response = {
 				data: {
 					params: formatAuctionParamsForResponse(auctionParams),

src/publishers/dlobPublisher.ts

@@ -43,6 +43,10 @@ import { handleHealthCheck } from '../core/middleware';
 import { setGlobalDispatcher, Agent } from 'undici';
 import { Metrics } from '../core/metricsV2';
 import { OrderSubscriberFiltered } from '../dlob-subscriber/OrderSubscriberFiltered';
+import {
+	FillQualityAnalyticsRepository,
+	TakerFillVsOracleBpsRedisResult,
+} from '../athena/repositories/fillQualityAnalytics';
 
 setGlobalDispatcher(
 	new Agent({
@@ -93,6 +97,10 @@ const tobStuckGauge = metricsV2.addGauge(
 	'tob_stuck_duration',
 	'Duration TOB has been stuck for each market'
 );
+const takerFillBpsFromOracleGauge = metricsV2.addGauge(
+	'taker_fill_bps_from_oracle',
+	'Taker fill BPS from oracle by market, side, and cohort'
+);
 metricsV2.finalizeObservables();
 
 //@ts-ignore
@@ -109,7 +117,7 @@ const endpoint = process.env.ENDPOINT;
 const grpcEndpoint = useGrpc
 	? process.env.GRPC_ENDPOINT ?? endpoint + `/${token}`
 	: '';
-const grpcClient = process.env.GRPC_CLIENT ?? 'yellowstone'
+const grpcClient = process.env.GRPC_CLIENT ?? 'yellowstone';
 
 const wsEndpoint = process.env.WS_ENDPOINT;
 const useOrderSubscriber =
@@ -122,6 +130,16 @@ const WS_FALLBACK_FETCH_INTERVAL = 60_000;
 const KILLSWITCH_SLOT_DIFF_THRESHOLD =
 	parseInt(process.env.KILLSWITCH_SLOT_DIFF_THRESHOLD) || 200;
 
+// Fill Quality Analytics configuration
+const ENABLE_FILL_QUALITY_ANALYTICS =
+	process.env.ENABLE_FILL_QUALITY_ANALYTICS?.toLowerCase() === 'true';
+const FILL_QUALITY_ANALYTICS_INTERVAL =
+	parseInt(process.env.FILL_QUALITY_ANALYTICS_INTERVAL) || 300_000; // 5 minutes default
+const FILL_QUALITY_ANALYTICS_LOOKBACK_MS =
+	parseInt(process.env.FILL_QUALITY_ANALYTICS_LOOKBACK_MS) || 86_400_000; // 24 hours default
+const FILL_QUALITY_ANALYTICS_SMOOTHING_MINUTES =
+	parseInt(process.env.FILL_QUALITY_ANALYTICS_SMOOTHING_MINUTES) || 60;
+
 // TOB monitoring configuration - defaults to true if not set
 const ENABLE_TOB_MONITORING =
 	!process.env.ENABLE_TOB_MONITORING ||
@@ -169,6 +187,22 @@ if (ENABLE_TOB_MONITORING) {
 		`TOB Monitoring Markets: ${TOB_MONITORING_ENABLED_PERP_MARKETS.join(', ')}`
 	);
 }
+logger.info(
+	`Fill Quality Analytics: ${
+		ENABLE_FILL_QUALITY_ANALYTICS ? 'enabled' : 'disabled'
+	}`
+);
+if (ENABLE_FILL_QUALITY_ANALYTICS) {
+	logger.info(
+		`Fill Quality Analytics Interval: ${FILL_QUALITY_ANALYTICS_INTERVAL}ms`
+	);
+	logger.info(
+		`Fill Quality Analytics Lookback: ${FILL_QUALITY_ANALYTICS_LOOKBACK_MS}ms`
+	);
+	logger.info(
+		`Fill Quality Analytics Smoothing: ${FILL_QUALITY_ANALYTICS_SMOOTHING_MINUTES} minutes`
+	);
+}
 
 let MARKET_SUBSCRIBERS: SubscriberLookup = {};
 
@@ -504,8 +538,7 @@ const main = async () => {
 						'grpc.keepalive_timeout_ms': 1_000,
 						'grpc.keepalive_permit_without_calls': 1,
 					},
-					client: grpcClient
+					client: grpcClient,
-					
 				},
 				commitment: stateCommitment,
 			};
@@ -817,6 +850,196 @@ const main = async () => {
 	// Start TOB monitoring
 	setInterval(checkTobForStuckOrders, TOB_CHECK_INTERVAL);
 
+	// Track last known values for fill quality metrics (per market/side/cohort)
+	const lastFillQualityValues = new Map<string, number>();
+
+	// Helper function to update fill quality metric with null tracking
+	const updateFillQualityMetric = (
+		value: string | number | null | undefined,
+		marketIndex: string | number,
+		side: string,
+		cohort: string
+	) => {
+		const marketIndexStr = marketIndex.toString();
+		const key = `${marketIndexStr}:${side}:${cohort}`;
+		const isNull = value === null || value === undefined;
+
+		let valueToUse: number;
+		if (isNull) {
+			// Use last known value, or 0 if no previous value exists
+			valueToUse = lastFillQualityValues.get(key) || 0;
+			logger.debug(
+				`Null value for market ${marketIndexStr} ${side} ${cohort}, using last known value: ${valueToUse}`
+			);
+		} else {
+			// Use current value and update the last known value
+			valueToUse = Number(value);
+			lastFillQualityValues.set(key, valueToUse);
+		}
+
+		takerFillBpsFromOracleGauge.setLatestValue(valueToUse, {
+			marketIndex: marketIndexStr,
+			side,
+			cohort,
+			null: isNull ? 'true' : 'false',
+		});
+	};
+
+	// Fill Quality Analytics function - fetch and store in Redis
+	const fetchAndStoreFillQualityAnalytics = async () => {
+		if (!ENABLE_FILL_QUALITY_ANALYTICS) {
+			return;
+		}
+
+		try {
+			logger.info('Starting fill quality analytics fetch');
+			const startTime = Date.now();
+
+			const fillQualityRepo = FillQualityAnalyticsRepository();
+			const toMs = Date.now();
+			const fromMs = toMs - FILL_QUALITY_ANALYTICS_LOOKBACK_MS;
+
+			const results = await fillQualityRepo.getTakerFillVsOracleBps(
+				fromMs,
+				toMs,
+				9, // baseDecimals
+				FILL_QUALITY_ANALYTICS_SMOOTHING_MINUTES
+			);
+
+			logger.info(
+				`Fetched fill quality analytics for ${results.length} markets in ${
+					Date.now() - startTime
+				}ms`
+			);
+
+			// Store results in Redis - one key per market
+			const startRedisSet = Date.now();
+			for (const result of results) {
+				const redisKey = `taker_fill_vs_oracle_bps:market:${result.MarketIndex}`;
+				const redisValue = JSON.stringify({
+					marketIndex: result.MarketIndex,
+					takerBuyBpsFromOracle: {
+						all: result.TakerBuyBpsFromOracle_ALL,
+						'1e0': result.TakerBuyBpsFromOracle_1e0,
+						'1e3': result.TakerBuyBpsFromOracle_1e3,
+						'1e4': result.TakerBuyBpsFromOracle_1e4,
+						'1e5': result.TakerBuyBpsFromOracle_1e5,
+						'1e6': result.TakerBuyBpsFromOracle_1e6,
+					},
+					takerSellBpsFromOracle: {
+						all: result.TakerSellBpsFromOracle_ALL,
+						'1e0': result.TakerSellBpsFromOracle_1e0,
+						'1e3': result.TakerSellBpsFromOracle_1e3,
+						'1e4': result.TakerSellBpsFromOracle_1e4,
+						'1e5': result.TakerSellBpsFromOracle_1e5,
+						'1e6': result.TakerSellBpsFromOracle_1e6,
+					},
+					updatedAtTs: Date.now(),
+				} as TakerFillVsOracleBpsRedisResult);
+
+				await redisClient.set(redisKey, redisValue);
+
+				updateFillQualityMetric(
+					result.TakerBuyBpsFromOracle_ALL,
+					result.MarketIndex,
+					'buy',
+					'all'
+				);
+				updateFillQualityMetric(
+					result.TakerBuyBpsFromOracle_1e0,
+					result.MarketIndex,
+					'buy',
+					'1e0'
+				);
+				updateFillQualityMetric(
+					result.TakerBuyBpsFromOracle_1e3,
+					result.MarketIndex,
+					'buy',
+					'1e3'
+				);
+				updateFillQualityMetric(
+					result.TakerBuyBpsFromOracle_1e4,
+					result.MarketIndex,
+					'buy',
+					'1e4'
+				);
+				updateFillQualityMetric(
+					result.TakerBuyBpsFromOracle_1e5,
+					result.MarketIndex,
+					'buy',
+					'1e5'
+				);
+				updateFillQualityMetric(
+					result.TakerBuyBpsFromOracle_1e6,
+					result.MarketIndex,
+					'buy',
+					'1e6'
+				);
+				updateFillQualityMetric(
+					result.TakerSellBpsFromOracle_ALL,
+					result.MarketIndex,
+					'sell',
+					'all'
+				);
+				updateFillQualityMetric(
+					result.TakerSellBpsFromOracle_1e0,
+					result.MarketIndex,
+					'sell',
+					'1e0'
+				);
+				updateFillQualityMetric(
+					result.TakerSellBpsFromOracle_1e3,
+					result.MarketIndex,
+					'sell',
+					'1e3'
+				);
+				updateFillQualityMetric(
+					result.TakerSellBpsFromOracle_1e4,
+					result.MarketIndex,
+					'sell',
+					'1e4'
+				);
+				updateFillQualityMetric(
+					result.TakerSellBpsFromOracle_1e5,
+					result.MarketIndex,
+					'sell',
+					'1e5'
+				);
+				updateFillQualityMetric(
+					result.TakerSellBpsFromOracle_1e6,
+					result.MarketIndex,
+					'sell',
+					'1e6'
+				);
+
+				logger.debug(
+					`Stored fill quality analytics for market ${result.MarketIndex}`
+				);
+			}
+			logger.info(
+				`Successfully stored fill quality analytics for ${
+					results.length
+				} markets in Redis in ${Date.now() - startRedisSet}ms`
+			);
+		} catch (error) {
+			logger.error('Error fetching/storing fill quality analytics:', error);
+		}
+	};
+
+	// Run fill quality analytics fetch immediately on startup, then on interval
+	if (ENABLE_FILL_QUALITY_ANALYTICS) {
+		fetchAndStoreFillQualityAnalytics().catch((error) => {
+			logger.error('Initial fill quality analytics fetch failed:', error);
+		});
+		setInterval(
+			fetchAndStoreFillQualityAnalytics,
+			FILL_QUALITY_ANALYTICS_INTERVAL
+		);
+		logger.info(
+			`Fill quality analytics will run every ${FILL_QUALITY_ANALYTICS_INTERVAL}ms`
+		);
+	}
+
 	const handleStartup = async (_req, res, _next) => {
 		if (driftClient.isSubscribed && dlobProvider.size() > 0) {
 			res.writeHead(200);

src/utils/utils.ts

@@ -34,6 +34,7 @@ import { wsMarketArgs } from 'src/dlob-subscriber/DLOBSubscriberIO';
 import { DEFAULT_AUCTION_PARAMS, MID_MAJOR_MARKETS } from './constants';
 import { AuctionParamArgs } from './types';
 import { COMMON_MATH, ENUM_UTILS } from '@drift/common';
+import { TakerFillVsOracleBpsRedisResult } from '../athena/repositories/fillQualityAnalytics';
 
 export const GROUPING_OPTIONS = [1, 10, 100, 500, 1000];
 export const GROUPING_DEPENDENCIES = {
@@ -312,10 +313,17 @@ export function publishGroupings(
 			asks: aggregatedAsks,
 		});
 
-
+		if (
-		if(['SOL-PERP', 'BTC-PERP', 'ETH-PERP'].includes(l2Formatted_grouped20.marketName) && aggregatedBids.length !== 20 || aggregatedAsks.length !== 20) {
+			(['SOL-PERP', 'BTC-PERP', 'ETH-PERP'].includes(
-			logger.error(`Error aggregating dlob levels: group=${group}, bids=${fullAggregatedBids.length}, asks=${fullAggregatedAsks.length}`)
+				l2Formatted_grouped20.marketName
-			logger.error(`Response: ${JSON.stringify(l2Formatted_grouped20)}`)
+			) &&
+				aggregatedBids.length !== 20) ||
+			aggregatedAsks.length !== 20
+		) {
+			logger.error(
+				`Error aggregating dlob levels: group=${group}, bids=${fullAggregatedBids.length}, asks=${fullAggregatedAsks.length}`
+			);
+			logger.error(`Response: ${JSON.stringify(l2Formatted_grouped20)}`);
 		}
 
 		redisClient.publish(
@@ -911,6 +919,8 @@ export const getEstimatedPrices = async (
  * @param driftClient - DriftClient instance (optional, for price calculation)
  * @param fetchFromRedis - Redis fetch function (optional, for price calculation)
  * @param selectMostRecentBySlot - Slot selection function (optional, for price calculation)
+ * @param fillQualityInfo - Fill quality analytics data (version 2 only)
+ * @param apiVersion - API version (1 or 2)
  * @returns Object formatted for deriveMarketOrderParams function or error response
  */
 export const mapToMarketOrderParams = async (
@@ -920,7 +930,9 @@ export const mapToMarketOrderParams = async (
 		key: string,
 		selectionCriteria: (responses: any) => any
 	) => Promise<any>,
-	selectMostRecentBySlot?: (responses: any[]) => any
+	selectMostRecentBySlot?: (responses: any[]) => any,
+	fillQualityInfo?: TakerFillVsOracleBpsRedisResult,
+	apiVersion: number = 1
 ): Promise<{
 	success: boolean;
 	data?: {
@@ -960,6 +972,15 @@ export const mapToMarketOrderParams = async (
 	let estimatedPrices;
 	let processedSlippageTolerance = params.slippageTolerance;
 
+	// Track debug info for logging
+	const debugInfo = {
+		originalOraclePrice: null as string | null,
+		adjustedOraclePrice: null as string | null,
+		isCrossed: false,
+		fillQualityBps: null as number | null,
+		appliedV2Adjustment: false,
+	};
+
 	if (driftClient && fetchFromRedis && selectMostRecentBySlot) {
 		// Get L2 orderbook data using the utility function
 		const redisL2 = await fetchL2FromRedis(
@@ -980,6 +1001,73 @@ export const mapToMarketOrderParams = async (
 			redisL2
 		);
 
+		// Store original oracle for debugging
+		debugInfo.originalOraclePrice = estimatedPrices.oraclePrice.toString();
+
+		// VERSION 2: Adjust oracle price based on fill quality when orderbook is crossed
+		if (apiVersion === 2 && fillQualityInfo && redisL2) {
+			try {
+				// Convert raw L2 to formatted L2 for cross detection
+				const l2Formatted = convertRawL2ToBN(redisL2);
+
+				const isSpot = isVariant(marketType, 'spot');
+				const oracleData = isSpot
+					? driftClient.getOracleDataForSpotMarket(params.marketIndex)
+					: driftClient.getOracleDataForPerpMarket(params.marketIndex);
+				const oraclePrice = oracleData.price ?? ZERO;
+
+				// Detect if orderbook is crossed
+				const spreadInfo = COMMON_MATH.calculateSpreadBidAskMark(
+					l2Formatted,
+					oraclePrice
+				);
+
+				const isCrossed =
+					spreadInfo.bestBidPrice &&
+					spreadInfo.bestAskPrice &&
+					spreadInfo.bestBidPrice.gte(spreadInfo.bestAskPrice);
+
+				debugInfo.isCrossed = isCrossed;
+
+				if (isCrossed) {
+					// Get fill quality metric based on direction
+					const fillQualityBpsStr =
+						direction === PositionDirection.LONG
+							? fillQualityInfo.takerBuyBpsFromOracle?.all
+							: fillQualityInfo.takerSellBpsFromOracle?.all;
+
+					if (
+						fillQualityBpsStr &&
+						fillQualityBpsStr !== 'null' &&
+						fillQualityBpsStr !== null
+					) {
+						const fillQualityBps = Math.round(
+							parseFloat(fillQualityBpsStr) * 100
+						);
+						debugInfo.fillQualityBps = fillQualityBps;
+
+						if (!isNaN(fillQualityBps)) {
+							const adjustment = oraclePrice
+								.muln(fillQualityBps)
+								.divn(10000 * 100); // adjustmentFactor is in bps
+							const adjustedOraclePrice = oraclePrice.add(adjustment);
+
+							// Update the oracle price in estimatedPrices
+							estimatedPrices.oraclePrice = adjustedOraclePrice;
+							debugInfo.adjustedOraclePrice = adjustedOraclePrice.toString();
+							debugInfo.appliedV2Adjustment = true;
+						}
+					}
+				}
+			} catch (error) {
+				logger.warn(
+					'Version 2: Failed to apply fill quality adjustment, using standard oracle:',
+					error
+				);
+				// Fall through to use unadjusted oracle
+			}
+		}
+
 		// Handle dynamic slippage tolerance calculation if needed
 		if (params.slippageTolerance === undefined) {
 			// Convert raw L2 to formatted L2 for slippage calculation
@@ -1028,6 +1116,62 @@ export const mapToMarketOrderParams = async (
 		baseAmount = amount.mul(BASE_PRECISION).div(estimatedPrices.entryPrice);
 	}
 
+	// Comprehensive debug logging
+	logger.info(
+		JSON.stringify({
+			event: 'auction_params_calculated',
+			requestParams: {
+				marketIndex: params.marketIndex,
+				marketType: params.marketType,
+				direction: direction === PositionDirection.LONG ? 'long' : 'short',
+				amount: params.amount,
+				assetType: params.assetType,
+				slippageTolerance: params.slippageTolerance,
+				apiVersion,
+			},
+			priceDiscovery: {
+				originalOraclePrice: debugInfo.originalOraclePrice,
+				finalOraclePrice: estimatedPrices.oraclePrice.toString(),
+				bestPrice: estimatedPrices.bestPrice.toString(),
+				entryPrice: estimatedPrices.entryPrice.toString(),
+				worstPrice: estimatedPrices.worstPrice.toString(),
+				markPrice: estimatedPrices.markPrice.toString(),
+				priceImpactBps: estimatedPrices.priceImpact.toString(),
+			},
+			v2CrossDetection:
+				apiVersion === 2
+					? {
+							isCrossed: debugInfo.isCrossed,
+							fillQualityBps: debugInfo.fillQualityBps,
+							adjustedOraclePrice: debugInfo.adjustedOraclePrice,
+							appliedAdjustment: debugInfo.appliedV2Adjustment,
+							fillQualityData: fillQualityInfo
+								? {
+										takerBuyBpsAll: fillQualityInfo.takerBuyBpsFromOracle?.all,
+										takerSellBpsAll:
+											fillQualityInfo.takerSellBpsFromOracle?.all,
+								  }
+								: null,
+					  }
+					: undefined,
+			auctionConfig: {
+				duration: params.auctionDuration,
+				startPriceOffset: params.auctionStartPriceOffset,
+				startPriceOffsetFrom: params.auctionStartPriceOffsetFrom,
+				endPriceOffset: params.auctionEndPriceOffset,
+				endPriceOffsetFrom: params.auctionEndPriceOffsetFrom,
+				slippageTolerance: processedSlippageTolerance,
+				isOracleOrder: params.isOracleOrder,
+				reduceOnly: params.reduceOnly ?? false,
+				allowInfSlippage: params.allowInfSlippage ?? false,
+			},
+			calculatedValues: {
+				baseAmount: baseAmount.toString(),
+				slippageToleranceFinal: processedSlippageTolerance,
+			},
+		})
+	);
+
 	return {
 		success: true,
 		data: {