feat(metrics): initial ai impl

This commit is contained in:
2026-03-06 19:32:12 +01:00
parent ba30d6e097
commit 527155a0c1
25 changed files with 1576 additions and 195 deletions

View File

@@ -0,0 +1,787 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(255, 211, 102, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 1,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [
{
"options": {
"0": {
"text": "Down"
},
"1": {
"text": "Up"
}
},
"type": "value"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "value"
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "max(up{job=\"web\"})",
"refId": "A"
}
],
"title": "Web",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [
{
"options": {
"0": {
"text": "Down"
},
"1": {
"text": "Up"
}
},
"type": "value"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 4,
"y": 0
},
"id": 2,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "value"
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "max(up{job=\"chat\"})",
"refId": "A"
}
],
"title": "Chat",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [
{
"options": {
"0": {
"text": "Down"
},
"1": {
"text": "Up"
}
},
"type": "value"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 8,
"y": 0
},
"id": 3,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "value"
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "max(up{job=\"mediamtx\"})",
"refId": "A"
}
],
"title": "MediaMTX",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [
{
"options": {
"0": {
"text": "Down"
},
"1": {
"text": "Up"
}
},
"type": "value"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 12,
"y": 0
},
"id": 4,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "value"
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "max(pg_up)",
"refId": "A"
}
],
"title": "Postgres",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [
{
"options": {
"0": {
"text": "Down"
},
"1": {
"text": "Up"
}
},
"type": "value"
}
],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "red",
"value": null
},
{
"color": "green",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 4,
"x": 16,
"y": 0
},
"id": 5,
"options": {
"colorMode": "background",
"graphMode": "none",
"justifyMode": "center",
"orientation": "auto",
"reduceOptions": {
"calcs": ["lastNotNull"],
"fields": "",
"values": false
},
"textMode": "value"
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "max(redis_up)",
"refId": "A"
}
],
"title": "Redis",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 4
},
"id": 6,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "sum by (region) (hctv_web_live_streams)",
"legendFormat": "{{region}}",
"refId": "A"
}
],
"title": "Live Streams by Region",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "none"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 4
},
"id": 7,
"options": {
"legend": {
"displayMode": "list",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "hctv_web_active_viewers",
"legendFormat": "active viewers",
"refId": "A"
},
{
"expr": "hctv_chat_websocket_connections",
"legendFormat": "chat sockets",
"refId": "B"
}
],
"title": "Audience Pulse",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 4
},
"id": 8,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "sum(rate(hctv_chat_messages_delivered_total[5m]))",
"legendFormat": "delivered messages",
"refId": "A"
},
{
"expr": "sum(rate(hctv_chat_incoming_messages_total[5m]))",
"legendFormat": "all inbound frames",
"refId": "B"
}
],
"title": "Chat Throughput",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "ops"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 12
},
"id": 9,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "sum by (action) (rate(hctv_chat_moderation_actions_total[15m]))",
"legendFormat": "{{action}}",
"refId": "A"
},
{
"expr": "sum by (reason) (rate(hctv_chat_moderation_blocks_total[15m]))",
"legendFormat": "blocked: {{reason}}",
"refId": "B"
}
],
"title": "Moderation Load",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "reqps"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 12
},
"id": 10,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "sum by (outcome) (rate(hctv_web_mediamtx_auth_requests_total[5m]))",
"legendFormat": "{{outcome}}",
"refId": "A"
}
],
"title": "Media Auth Outcomes",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 12
},
"id": 11,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "histogram_quantile(0.95, sum by (le, job) (rate(hctv_web_background_job_duration_seconds_bucket[15m])))",
"legendFormat": "{{job}} p95",
"refId": "A"
}
],
"title": "Background Job Latency P95",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 0,
"y": 20
},
"id": 12,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "pg_stat_database_numbackends{datname=~\"hctv|postgres\"}",
"legendFormat": "database connections",
"refId": "A"
},
{
"expr": "rate(pg_stat_database_xact_commit{datname=~\"hctv|postgres\"}[5m])",
"legendFormat": "commits / sec",
"refId": "B"
}
],
"title": "Postgres Activity",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 8,
"y": 20
},
"id": 13,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "redis_memory_used_bytes",
"legendFormat": "redis memory",
"refId": "A"
},
{
"expr": "redis_connected_clients",
"legendFormat": "connected clients",
"refId": "B"
}
],
"title": "Redis Footprint",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "prometheus"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"unit": "bytes"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 8,
"x": 16,
"y": 20
},
"id": 14,
"options": {
"legend": {
"displayMode": "table",
"placement": "bottom"
},
"tooltip": {
"mode": "multi"
}
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "hctv_web_process_resident_memory_bytes",
"legendFormat": "web memory",
"refId": "A"
},
{
"expr": "hctv_chat_process_resident_memory_bytes",
"legendFormat": "chat memory",
"refId": "B"
},
{
"expr": "process_resident_memory_bytes{job=\"mediamtx\"}",
"legendFormat": "mediamtx memory",
"refId": "C"
}
],
"title": "Service Memory",
"type": "timeseries"
}
],
"refresh": "30s",
"schemaVersion": 39,
"style": "dark",
"tags": ["hackclub.tv", "observability"],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "HackClub.tv Overview",
"uid": "hctv-overview",
"version": 1,
"weekStart": ""
}

View File

@@ -0,0 +1,10 @@
apiVersion: 1
providers:
- name: HackClubTV
folder: HackClub TV
type: file
disableDeletion: false
editable: true
options:
path: /var/lib/grafana/dashboards

View File

@@ -0,0 +1,10 @@
apiVersion: 1
datasources:
- name: Prometheus
uid: prometheus
type: prometheus
access: proxy
url: http://prometheus:9090
isDefault: true
editable: false

View File

@@ -0,0 +1,39 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
scrape_configs:
- job_name: web
metrics_path: /api/metrics
static_configs:
- targets:
- host.docker.internal:3000
- job_name: chat
metrics_path: /metrics
static_configs:
- targets:
- host.docker.internal:8000
- job_name: mediamtx
metrics_path: /metrics
static_configs:
- targets:
- mediamtx:9998
- job_name: redis
metrics_path: /metrics
static_configs:
- targets:
- redis-exporter:9121
- job_name: postgres
metrics_path: /metrics
static_configs:
- targets:
- postgres-exporter:9187
- job_name: prometheus
static_configs:
- targets:
- localhost:9090

View File

@@ -0,0 +1,39 @@
global:
scrape_interval: 15s
evaluation_interval: 15s
scrape_configs:
- job_name: web
metrics_path: /api/metrics
static_configs:
- targets:
- hctv:3000
- job_name: chat
metrics_path: /metrics
static_configs:
- targets:
- chat:8000
- job_name: mediamtx
metrics_path: /metrics
static_configs:
- targets:
- mediamtx:9998
- job_name: redis
metrics_path: /metrics
static_configs:
- targets:
- redis-exporter:9121
- job_name: postgres
metrics_path: /metrics
static_configs:
- targets:
- postgres-exporter:9187
- job_name: prometheus
static_configs:
- targets:
- localhost:9090