Skip to content

Commit b58e212

Browse files
Ferryx349Ferryx349
committed
feat(admin): add Prometheus-backed /admin/metrics SSE endpoint
Signed-off-by: ABHAY PANDEY <pandeyabhay967@gmail.com>
1 parent 936978d commit b58e212

11 files changed

Lines changed: 476 additions & 3 deletions

File tree

‎.changeset/admin-metrics-sse-endpoint.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
---
2+
"nostream": minor
3+
---
4+
5+
feat: add Prometheus-backed admin metrics SSE endpoint

‎.env.example‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,11 @@ WORKER_COUNT=2 # Defaults to CPU count. Use 1 or 2 for local testing.
3737
# OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318
3838
# OTEL_METRIC_EXPORT_INTERVAL_MS=10000
3939

40+
# --- OBSERVABILITY (Prometheus for Admin Metrics) ---
41+
# PROMETHEUS_URL=http://127.0.0.1:9090
42+
# PROMETHEUS_QUERY_TIMEOUT_MS=5000
43+
# ADMIN_METRICS_SSE_INTERVAL_MS=5000
44+
4045
# --- RELAY PRIVATE KEY (Optional) ---
4146
# RELAY_PRIVATE_KEY=your_hex_private_key
4247

‎docker-compose.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ services:
6060
OTEL_SERVICE_NAME: ${OTEL_SERVICE_NAME:-nostream}
6161
OTEL_EXPORTER_OTLP_ENDPOINT: ${OTEL_EXPORTER_OTLP_ENDPOINT:-http://otel-collector:4318}
6262
OTEL_METRIC_EXPORT_INTERVAL_MS: ${OTEL_METRIC_EXPORT_INTERVAL_MS:-10000}
63+
PROMETHEUS_URL: ${PROMETHEUS_URL:-http://prometheus:9090}
64+
ADMIN_METRICS_SSE_INTERVAL_MS: ${ADMIN_METRICS_SSE_INTERVAL_MS:-5000}
6365
# Enable DEBUG for troubleshooting. Examples:
6466
# DEBUG: "primary:*"
6567
# DEBUG: "worker:*"

‎src/controllers/admin/get-metrics-controller.ts‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import { Request, Response } from 'express'
2+
3+
import { IController } from '../../@types/controllers'
4+
import { createLogger } from '../../factories/logger-factory'
5+
import { collectAdminMetricsSnapshot, getAdminMetricsSseIntervalMs } from '../../utils/admin-metrics'
6+
7+
const logger = createLogger('get-admin-metrics-controller')
8+
9+
export class GetAdminMetricsController implements IController {
10+
public async handleRequest(request: Request, response: Response): Promise<void> {
11+
response.status(200)
12+
response.setHeader('content-type', 'text/event-stream; charset=utf-8')
13+
response.setHeader('cache-control', 'no-cache, no-transform')
14+
response.setHeader('connection', 'keep-alive')
15+
response.flushHeaders()
16+
17+
let closed = false
18+
19+
const sendSnapshot = async () => {
20+
if (closed || response.writableEnded) {
21+
return
22+
}
23+
24+
try {
25+
const snapshot = await collectAdminMetricsSnapshot()
26+
response.write(`data: ${JSON.stringify(snapshot)}\n\n`)
27+
} catch (error) {
28+
logger.warn('failed to collect admin metrics snapshot: %o', error)
29+
if (!closed && !response.writableEnded) {
30+
response.write(`event: error\ndata: ${JSON.stringify({ error: 'failed to collect metrics' })}\n\n`)
31+
}
32+
}
33+
}
34+
35+
await sendSnapshot()
36+
37+
const intervalMs = getAdminMetricsSseIntervalMs()
38+
const timer = setInterval(() => {
39+
void sendSnapshot()
40+
}, intervalMs)
41+
42+
request.on('close', () => {
43+
closed = true
44+
clearInterval(timer)
45+
})
46+
}
47+
}

‎src/factories/controllers/get-admin-metrics-controller-factory.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import { GetAdminMetricsController } from '../../controllers/admin/get-metrics-controller'
2+
import { IController } from '../../@types/controllers'
3+
4+
export const createGetAdminMetricsController = (): IController => {
5+
return new GetAdminMetricsController()
6+
}

‎src/routes/admin/index.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import { json, Router } from 'express'
22

33
import { createGetAdminHealthController } from '../../factories/controllers/get-admin-health-controller-factory'
4+
import { createGetAdminMetricsController } from '../../factories/controllers/get-admin-metrics-controller-factory'
45
import { createGetAdminSessionController } from '../../factories/controllers/get-admin-session-controller-factory'
56
import { createPostAdminLoginController } from '../../factories/controllers/post-admin-login-controller-factory'
67
import { adminAuthMiddleware } from '../../handlers/request-handlers/admin-auth-middleware'
@@ -21,5 +22,6 @@ router.use(adminEnabledMiddleware)
2122
router.post('/login', adminLoginRateLimitMiddleware, json(), withAdminController(createPostAdminLoginController))
2223
router.get('/session', adminRateLimitMiddleware, adminAuthMiddleware, withAdminController(createGetAdminSessionController))
2324
router.get('/health', adminRateLimitMiddleware, adminAuthMiddleware, withAdminController(createGetAdminHealthController))
25+
router.get('/metrics', adminRateLimitMiddleware, adminAuthMiddleware, withAdminController(createGetAdminMetricsController))
2426

2527
export default router

‎src/utils/admin-metrics.ts‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
import { AdminDependencyHealth, collectAdminHealthSnapshot } from './admin-health'
2+
import { queryPrometheusInstant } from './prometheus-client'
3+
4+
export interface AdminMetricsValues {
5+
eventsPerSecond: number
6+
eventsRejectedPerSecond: number
7+
eventsAcceptedTotal: number
8+
eventsRejectedTotal: number
9+
activeConnections: number
10+
cpuLoadPercent: number
11+
memoryUsedMb: number
12+
}
13+
14+
export interface AdminMetricsSnapshot {
15+
timestamp: number
16+
status: 'ok' | 'degraded' | 'unavailable'
17+
metrics: AdminMetricsValues
18+
health: {
19+
database: AdminDependencyHealth
20+
redis: AdminDependencyHealth
21+
}
22+
prometheus: {
23+
available: boolean
24+
error?: string
25+
}
26+
}
27+
28+
const PROMQL = {
29+
eventsAcceptedRate: 'sum(rate(nostream_events_accepted_total[1m]))',
30+
eventsRejectedRate: 'sum(rate(nostream_events_rejected_total[1m]))',
31+
eventsAcceptedTotal: 'sum(nostream_events_accepted_total)',
32+
eventsRejectedTotal: 'sum(nostream_events_rejected_total)',
33+
activeConnections: 'sum(nostream_websocket_connections)',
34+
cpuLoadPercent: 'avg(nostream_process_cpu_load_percent)',
35+
memoryUsedMb: 'sum(nostream_process_memory_used_mb)',
36+
} as const
37+
38+
const toMetricValue = (value: number | undefined): number => {
39+
return value ?? 0
40+
}
41+
42+
export const collectAdminMetricsSnapshot = async (): Promise<AdminMetricsSnapshot> => {
43+
const [
44+
health,
45+
eventsAcceptedRate,
46+
eventsRejectedRate,
47+
eventsAcceptedTotal,
48+
eventsRejectedTotal,
49+
activeConnections,
50+
cpuLoadPercent,
51+
memoryUsedMb,
52+
] = await Promise.all([
53+
collectAdminHealthSnapshot(),
54+
queryPrometheusInstant(PROMQL.eventsAcceptedRate),
55+
queryPrometheusInstant(PROMQL.eventsRejectedRate),
56+
queryPrometheusInstant(PROMQL.eventsAcceptedTotal),
57+
queryPrometheusInstant(PROMQL.eventsRejectedTotal),
58+
queryPrometheusInstant(PROMQL.activeConnections),
59+
queryPrometheusInstant(PROMQL.cpuLoadPercent),
60+
queryPrometheusInstant(PROMQL.memoryUsedMb),
61+
])
62+
63+
const prometheusAvailable =
64+
eventsAcceptedRate !== undefined ||
65+
eventsRejectedRate !== undefined ||
66+
eventsAcceptedTotal !== undefined ||
67+
eventsRejectedTotal !== undefined ||
68+
activeConnections !== undefined ||
69+
cpuLoadPercent !== undefined ||
70+
memoryUsedMb !== undefined
71+
72+
const metricsHealthy = health.database.ok && health.redis.ok
73+
let status: AdminMetricsSnapshot['status'] = 'ok'
74+
75+
if (!prometheusAvailable) {
76+
status = 'unavailable'
77+
} else if (!metricsHealthy) {
78+
status = 'degraded'
79+
}
80+
81+
return {
82+
timestamp: Date.now(),
83+
status,
84+
metrics: {
85+
eventsPerSecond: toMetricValue(eventsAcceptedRate),
86+
eventsRejectedPerSecond: toMetricValue(eventsRejectedRate),
87+
eventsAcceptedTotal: toMetricValue(eventsAcceptedTotal),
88+
eventsRejectedTotal: toMetricValue(eventsRejectedTotal),
89+
activeConnections: toMetricValue(activeConnections),
90+
cpuLoadPercent: toMetricValue(cpuLoadPercent),
91+
memoryUsedMb: toMetricValue(memoryUsedMb),
92+
},
93+
health: {
94+
database: health.database,
95+
redis: health.redis,
96+
},
97+
prometheus: {
98+
available: prometheusAvailable,
99+
...(prometheusAvailable ? {} : { error: 'Prometheus query returned no data' }),
100+
},
101+
}
102+
}
103+
104+
export const getAdminMetricsSseIntervalMs = (): number => {
105+
const intervalCandidate = Number(process.env.ADMIN_METRICS_SSE_INTERVAL_MS || 5000)
106+
107+
if (!Number.isFinite(intervalCandidate) || intervalCandidate < 1000) {
108+
return 5000
109+
}
110+
111+
return intervalCandidate
112+
}

‎src/utils/prometheus-client.ts‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
import axios from 'axios'
2+
3+
import { createLogger } from '../factories/logger-factory'
4+
5+
const logger = createLogger('prometheus-client')
6+
7+
const DEFAULT_PROMETHEUS_URL = 'http://127.0.0.1:9090'
8+
9+
interface PrometheusInstantQueryResponse {
10+
status: 'success' | 'error'
11+
data?: {
12+
resultType: string
13+
result: Array<{
14+
value?: [number, string]
15+
}>
16+
}
17+
error?: string
18+
errorType?: string
19+
}
20+
21+
export const getPrometheusBaseUrl = (): string => {
22+
const configured = process.env.PROMETHEUS_URL?.trim()
23+
if (!configured) {
24+
return DEFAULT_PROMETHEUS_URL
25+
}
26+
27+
return configured.replace(/\/+$/, '')
28+
}
29+
30+
export const parsePrometheusInstantQueryScalar = (response: PrometheusInstantQueryResponse): number | undefined => {
31+
if (response.status !== 'success') {
32+
return undefined
33+
}
34+
35+
const rawValue = response.data?.result?.[0]?.value?.[1]
36+
if (rawValue === undefined) {
37+
return undefined
38+
}
39+
40+
const parsed = Number(rawValue)
41+
return Number.isFinite(parsed) ? parsed : undefined
42+
}
43+
44+
export const queryPrometheusInstant = async (query: string): Promise<number | undefined> => {
45+
const baseUrl = getPrometheusBaseUrl()
46+
47+
try {
48+
const response = await axios.get<PrometheusInstantQueryResponse>(`${baseUrl}/api/v1/query`, {
49+
params: { query },
50+
timeout: Number(process.env.PROMETHEUS_QUERY_TIMEOUT_MS || 5000),
51+
validateStatus: () => true,
52+
})
53+
54+
if (response.status !== 200) {
55+
logger.warn('prometheus query failed with HTTP %d: %s', response.status, query)
56+
return undefined
57+
}
58+
59+
if (response.data.status !== 'success') {
60+
logger.warn('prometheus query error for %s: %s', query, response.data.error ?? response.data.errorType ?? 'unknown')
61+
return undefined
62+
}
63+
64+
return parsePrometheusInstantQueryScalar(response.data)
65+
} catch (error) {
66+
logger.warn('prometheus query request failed for %s: %o', query, error)
67+
return undefined
68+
}
69+
}

‎test/unit/routes/admin.spec.ts‎

Lines changed: 29 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import express from 'express'
44
import Sinon from 'sinon'
55

66
import * as getAdminHealthControllerFactory from '../../../src/factories/controllers/get-admin-health-controller-factory'
7+
import * as getAdminMetricsControllerFactory from '../../../src/factories/controllers/get-admin-metrics-controller-factory'
78
import { hashAdminPassword } from '../../../src/utils/admin-password'
89
import * as adminRateLimitMiddleware from '../../../src/handlers/request-handlers/admin-rate-limit-middleware'
910
import * as rateLimiterMiddleware from '../../../src/handlers/request-handlers/rate-limiter-middleware'
@@ -13,6 +14,7 @@ describe('admin router', () => {
1314
const originalSecret = process.env.SECRET
1415
const originalAdminPassword = process.env.ADMIN_PASSWORD
1516
let createGetAdminHealthControllerStub: Sinon.SinonStub
17+
let createGetAdminMetricsControllerStub: Sinon.SinonStub
1618
let createSettingsStub: Sinon.SinonStub
1719
let rateLimiterMiddlewareStub: Sinon.SinonStub
1820
let adminRateLimitMiddlewareStub: Sinon.SinonStub
@@ -48,6 +50,18 @@ describe('admin router', () => {
4850
)
4951
},
5052
} as any)
53+
createGetAdminMetricsControllerStub = Sinon.stub(
54+
getAdminMetricsControllerFactory,
55+
'createGetAdminMetricsController',
56+
).returns({
57+
handleRequest: async (_request: any, response: any) => {
58+
response
59+
.status(200)
60+
.setHeader('content-type', 'text/event-stream; charset=utf-8')
61+
.write('data: {"timestamp":1,"status":"ok"}\n\n')
62+
response.end()
63+
},
64+
} as any)
5165
createSettingsStub = Sinon.stub(settingsFactory, 'createSettings').returns(settings as any)
5266
const passthrough = async (_request: any, _response: any, next: any) => {
5367
next()
@@ -72,6 +86,7 @@ describe('admin router', () => {
7286

7387
const stopServer = async () => {
7488
createGetAdminHealthControllerStub?.restore()
89+
createGetAdminMetricsControllerStub?.restore()
7590
createSettingsStub?.restore()
7691
rateLimiterMiddlewareStub?.restore()
7792
adminRateLimitMiddlewareStub?.restore()
@@ -132,10 +147,12 @@ describe('admin router', () => {
132147

133148
const sessionResponse = await axios.get(`${baseUrl}/session`, { validateStatus: () => true })
134149
const healthResponse = await axios.get(`${baseUrl}/health`, { validateStatus: () => true })
150+
const metricsResponse = await axios.get(`${baseUrl}/metrics`, { validateStatus: () => true })
135151

136152
expect(sessionResponse.status).to.equal(401)
137153
expect(healthResponse.status).to.equal(401)
138-
expect(rateLimiterMiddlewareStub.calledTwice).to.be.true
154+
expect(metricsResponse.status).to.equal(401)
155+
expect(rateLimiterMiddlewareStub.callCount).to.equal(3)
139156
})
140157

141158
it('rejects invalid login credentials', async () => {
@@ -229,9 +246,18 @@ describe('admin router', () => {
229246
expect(healthResponse.status).to.equal(200)
230247
expect(healthResponse.data).to.include.keys('status', 'uptimeSeconds', 'worker', 'database', 'redis')
231248

232-
expect(rateLimiterMiddlewareStub.callCount).to.equal(3)
249+
const metricsResponse = await axios.get(`${baseUrl}/metrics`, {
250+
headers: { cookie },
251+
validateStatus: () => true,
252+
responseType: 'text',
253+
})
254+
expect(metricsResponse.status).to.equal(200)
255+
expect(metricsResponse.headers['content-type']).to.include('text/event-stream')
256+
expect(metricsResponse.data).to.include('"timestamp":1')
257+
258+
expect(rateLimiterMiddlewareStub.callCount).to.equal(4)
233259
expect(adminLoginRateLimitMiddlewareStub.calledOnce).to.be.true
234-
expect(adminRateLimitMiddlewareStub.calledTwice).to.be.true
260+
expect(adminRateLimitMiddlewareStub.callCount).to.equal(3)
235261
})
236262

237263
it('authenticates with passwordHash from settings', async () => {

0 commit comments

Comments
 (0)