rename folder and actually sanity check

This commit is contained in:
Nour Alharithi
2023-10-24 17:12:16 -07:00
parent 568ab5a239
commit 255ef1f7eb
7 changed files with 513 additions and 516 deletions

142
src/core/metrics.ts Normal file
View File

@@ -0,0 +1,142 @@
import { ObservableResult } from '@opentelemetry/api';
import { PrometheusExporter } from '@opentelemetry/exporter-prometheus';
import { logger } from '../utils/logger';
import {
ExplicitBucketHistogramAggregation,
InstrumentType,
MeterProvider,
View,
} from '@opentelemetry/sdk-metrics-base';
import { commitHash, driftEnv, endpoint, wsEndpoint } from '..';
/**
* Creates {count} buckets of size {increment} starting from {start}. Each bucket stores the count of values within its "size".
* @param start
* @param increment
* @param count
* @returns
*/
const createHistogramBuckets = (
start: number,
increment: number,
count: number
) => {
return new ExplicitBucketHistogramAggregation(
Array.from(new Array(count), (_, i) => start + i * increment)
);
};
enum METRIC_TYPES {
runtime_specs = 'runtime_specs',
endpoint_response_times_histogram = 'endpoint_response_times_histogram',
endpoint_response_status = 'endpoint_response_status',
health_status = 'health_status',
}
export enum HEALTH_STATUS {
Ok = 0,
StaleBulkAccountLoader,
UnhealthySlotSubscriber,
LivenessTesting,
}
const metricsPort =
parseInt(process.env.METRICS_PORT) || PrometheusExporter.DEFAULT_OPTIONS.port;
const { endpoint: defaultEndpoint } = PrometheusExporter.DEFAULT_OPTIONS;
const exporter = new PrometheusExporter(
{
port: metricsPort,
endpoint: defaultEndpoint,
},
() => {
logger.info(
`prometheus scrape endpoint started: http://localhost:${metricsPort}${defaultEndpoint}`
);
}
);
const meterName = 'dlob-meter';
const meterProvider = new MeterProvider({
views: [
new View({
instrumentName: METRIC_TYPES.endpoint_response_times_histogram,
instrumentType: InstrumentType.HISTOGRAM,
meterName,
aggregation: createHistogramBuckets(0, 20, 30),
}),
],
});
meterProvider.addMetricReader(exporter);
const meter = meterProvider.getMeter(meterName);
const runtimeSpecsGauge = meter.createObservableGauge(
METRIC_TYPES.runtime_specs,
{
description: 'Runtime sepcification of this program',
}
);
const bootTimeMs = Date.now();
runtimeSpecsGauge.addCallback((obs) => {
obs.observe(bootTimeMs, {
commit: commitHash,
driftEnv,
rpcEndpoint: endpoint,
wsEndpoint: wsEndpoint,
});
});
let healthStatus: HEALTH_STATUS = HEALTH_STATUS.Ok;
const healthStatusGauge = meter.createObservableGauge(
METRIC_TYPES.health_status,
{
description: 'Health status of this program',
}
);
healthStatusGauge.addCallback((obs: ObservableResult) => {
obs.observe(healthStatus, {});
});
const endpointResponseTimeHistogram = meter.createHistogram(
METRIC_TYPES.endpoint_response_times_histogram,
{
description: 'Duration of endpoint responses',
unit: 'ms',
}
);
const responseStatusCounter = meter.createCounter(
METRIC_TYPES.endpoint_response_status,
{
description: 'Count of endpoint responses by status code',
}
);
const startupTime = Date.now();
const handleHealthCheck = async (req, res, next) => {
try {
if (req.url === '/health' || req.url === '/') {
if (Date.now() > startupTime + 60 * 1000) {
healthStatus = HEALTH_STATUS.LivenessTesting;
res.writeHead(500);
res.end('Testing liveness test fail');
return;
}
// liveness check passed
healthStatus = HEALTH_STATUS.Ok;
res.writeHead(200);
res.end('OK');
} else {
res.writeHead(404);
res.end('Not found');
}
} catch (e) {
next(e);
}
};
export {
endpointResponseTimeHistogram,
responseStatusCounter,
handleHealthCheck,
};