diff --git a/apps/chat/src/index.ts b/apps/chat/src/index.ts index feaed8a..d43f266 100644 --- a/apps/chat/src/index.ts +++ b/apps/chat/src/index.ts @@ -13,6 +13,7 @@ import { recordChatModerationBlock, recordDeliveredChatMessage, recordDeliveredChatMessageBytes, + recordEmojiSearchResults, recordHistoryMessagesLoaded, recordIncomingChatMessage, recordUniqueChatter, @@ -772,6 +773,7 @@ app.get( const rawSearchTerm = (msg.searchTerm as string)?.trim() ?? ''; if (!rawSearchTerm || rawSearchTerm.length > 50) { ws.send(JSON.stringify({ type: 'emojiSearchResponse', results: [] })); + recordEmojiSearchResults('empty', 0); outcome = 'emoji_search_empty'; return; } @@ -802,6 +804,7 @@ app.get( results: results, }) ); + recordEmojiSearchResults('matched', results.length); outcome = 'emoji_search'; } else { ws.send( @@ -810,6 +813,7 @@ app.get( results: [], }) ); + recordEmojiSearchResults('no_match', 0); outcome = 'emoji_search_empty'; } } diff --git a/apps/chat/src/metrics.ts b/apps/chat/src/metrics.ts index 804498c..b862525 100644 --- a/apps/chat/src/metrics.ts +++ b/apps/chat/src/metrics.ts @@ -114,6 +114,14 @@ function createMetricsStore() { registers: [register], }); + const emojiSearchResults = new Histogram({ + name: 'hctv_chat_emoji_search_results', + help: 'Number of emoji search results returned per query.', + labelNames: ['outcome'], + buckets: [0, 1, 2, 5, 10, 25, 50, 100, 150], + registers: [register], + }); + const errors = new Counter({ name: 'hctv_chat_errors_total', help: 'Errors observed in the chat service grouped by phase.', @@ -126,6 +134,7 @@ function createMetricsStore() { deliveredMessageBytes, channelHistoryLoadedMessages, channelHistorySize, + emojiSearchResults, errors, inboundPayloadBytes, incomingMessages, @@ -232,3 +241,7 @@ export function recordChatModerationBlock(reason: string): void { export function recordChatError(phase: string): void { metrics.errors.inc({ phase }); } + +export function recordEmojiSearchResults(outcome: string, count: number): void { + metrics.emojiSearchResults.observe({ outcome }, count); +} diff --git a/observability/grafana/dashboards/hctv-overview.json b/observability/grafana/dashboards/hctv-overview.json index 694eef9..f2da2c7 100644 --- a/observability/grafana/dashboards/hctv-overview.json +++ b/observability/grafana/dashboards/hctv-overview.json @@ -15,6 +15,7 @@ } ] }, + "description": "Live health, streaming, chat, and infrastructure metrics for HackClub.tv", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -35,10 +36,10 @@ { "options": { "0": { - "text": "Down" + "text": "DOWN" }, "1": { - "text": "Up" + "text": "UP" } }, "type": "value" @@ -52,7 +53,7 @@ "value": null }, { - "color": "green", + "color": "#73BF69", "value": 1 } ] @@ -61,19 +62,21 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 3, + "h": 3, + "w": 4, "x": 0, "y": 0 }, "id": 1, "options": { - "colorMode": "background", + "colorMode": "background_solid", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -102,10 +105,10 @@ { "options": { "0": { - "text": "Down" + "text": "DOWN" }, "1": { - "text": "Up" + "text": "UP" } }, "type": "value" @@ -119,7 +122,7 @@ "value": null }, { - "color": "green", + "color": "#73BF69", "value": 1 } ] @@ -128,19 +131,21 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 3, - "x": 3, + "h": 3, + "w": 4, + "x": 4, "y": 0 }, "id": 2, "options": { - "colorMode": "background", + "colorMode": "background_solid", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -169,10 +174,10 @@ { "options": { "0": { - "text": "Down" + "text": "DOWN" }, "1": { - "text": "Up" + "text": "UP" } }, "type": "value" @@ -186,7 +191,7 @@ "value": null }, { - "color": "green", + "color": "#73BF69", "value": 1 } ] @@ -195,19 +200,21 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 3, - "x": 6, + "h": 3, + "w": 4, + "x": 8, "y": 0 }, "id": 3, "options": { - "colorMode": "background", + "colorMode": "background_solid", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -236,10 +243,10 @@ { "options": { "0": { - "text": "Down" + "text": "DOWN" }, "1": { - "text": "Up" + "text": "UP" } }, "type": "value" @@ -253,7 +260,7 @@ "value": null }, { - "color": "green", + "color": "#73BF69", "value": 1 } ] @@ -262,19 +269,21 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 3, - "x": 9, + "h": 3, + "w": 4, + "x": 12, "y": 0 }, "id": 4, "options": { - "colorMode": "background", + "colorMode": "background_solid", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -303,10 +312,10 @@ { "options": { "0": { - "text": "Down" + "text": "DOWN" }, "1": { - "text": "Up" + "text": "UP" } }, "type": "value" @@ -320,7 +329,7 @@ "value": null }, { - "color": "green", + "color": "#73BF69", "value": 1 } ] @@ -329,19 +338,21 @@ "overrides": [] }, "gridPos": { - "h": 4, - "w": 3, - "x": 12, + "h": 3, + "w": 4, + "x": 16, "y": 0 }, "id": 5, "options": { - "colorMode": "background", + "colorMode": "background_solid", "graphMode": "none", "justifyMode": "center", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, @@ -364,16 +375,33 @@ "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "#FF9830", + "value": 1 + }, + { + "color": "#73BF69", + "value": 5 + } + ] }, "unit": "short" }, "overrides": [] }, "gridPos": { - "h": 4, - "w": 3, - "x": 15, + "h": 3, + "w": 4, + "x": 20, "y": 0 }, "id": 6, @@ -383,95 +411,13 @@ "justifyMode": "center", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, - "textMode": "value_and_name" - }, - "targets": [ - { - "expr": "hctv_web_active_viewers", - "refId": "A" - } - ], - "title": "Active Viewers", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 18, - "y": 0 - }, - "id": 7, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "value_and_name" - }, - "targets": [ - { - "expr": "hctv_chat_websocket_connections", - "refId": "A" - } - ], - "title": "Chat Sockets", - "type": "stat" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 4, - "w": 3, - "x": 21, - "y": 0 - }, - "id": 8, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "value_and_name" + "textMode": "value" }, "targets": [ { @@ -482,6 +428,247 @@ "title": "Live Streams", "type": "stat" }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "#73BF69", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 3 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "hctv_web_active_viewers", + "refId": "A" + } + ], + "title": "Viewers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "#73BF69", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 3 + }, + "id": 8, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "targets": [ + { + "expr": "hctv_chat_websocket_connections", + "refId": "A" + } + ], + "title": "Chat Sockets", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "#73BF69", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "type": "stat", + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 3 + }, + "id": 49, + "targets": [ + { + "expr": "hctv_web_platform_inventory{entity=\"channels\"}", + "refId": "A" + } + ], + "title": "Channels" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "text", + "value": null + }, + { + "color": "#73BF69", + "value": 1 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "type": "stat", + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 3 + }, + "id": 50, + "targets": [ + { + "expr": "hctv_web_platform_inventory{entity=\"users\"}", + "refId": "A" + } + ], + "title": "Onboarded Users" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 100, + "panels": [], + "title": "Streams & Viewers", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -492,6 +679,19 @@ "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true + }, "unit": "none" }, "overrides": [] @@ -500,27 +700,31 @@ "h": 8, "w": 8, "x": 0, - "y": 4 + "y": 7 }, "id": 9, "options": { "legend": { "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "desc" } }, "targets": [ { "expr": "sum by (region) (hctv_web_live_streams)", - "legendFormat": "live {{region}}", + "legendFormat": "live \u2014 {{region}}", "refId": "A" }, { "expr": "sum by (region) (hctv_web_stream_paths_seen)", - "legendFormat": "paths {{region}}", + "legendFormat": "paths \u2014 {{region}}", "refId": "B" } ], @@ -537,6 +741,17 @@ "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "pointSize": 5, + "showPoints": "never", + "spanNulls": true + }, "unit": "none" }, "overrides": [] @@ -545,32 +760,37 @@ "h": 8, "w": 8, "x": 8, - "y": 4 + "y": 7 }, "id": 10, "options": { "legend": { "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "desc" } }, "targets": [ { "expr": "hctv_web_active_viewers", - "legendFormat": "active viewers", + "legendFormat": "total viewers", "refId": "A" }, { "expr": "hctv_web_hottest_stream_viewers", - "legendFormat": "top stream viewers", + "legendFormat": "top stream", "refId": "B" }, { "expr": "hctv_web_streams_with_viewers", - "legendFormat": "streams with viewers", + "legendFormat": "streams w/ viewers", "refId": "C" } ], @@ -587,6 +807,19 @@ "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + } + }, "unit": "none" }, "overrides": [] @@ -595,16 +828,20 @@ "h": 8, "w": 8, "x": 16, - "y": 4 + "y": 7 }, "id": 11, "options": { "legend": { "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "desc" } }, "targets": [ @@ -627,140 +864,14 @@ "color": { "mode": "palette-classic" }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 12 - }, - "id": 12, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "sum(rate(hctv_chat_messages_delivered_total[5m]))", - "legendFormat": "delivered messages", - "refId": "A" - }, - { - "expr": "sum(rate(hctv_chat_incoming_messages_total[5m]))", - "legendFormat": "inbound frames", - "refId": "B" - }, - { - "expr": "sum(rate(hctv_chat_message_bytes_delivered_total[5m]))", - "legendFormat": "message bytes/sec", - "refId": "C" - } - ], - "title": "Chat Throughput", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 12 - }, - "id": 13, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "sum by (channel) (hctv_chat_websocket_connections_by_channel)", - "legendFormat": "{{channel}}", - "refId": "A" - } - ], - "title": "Socket Load by Channel", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 12 - }, - "id": 14, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "sum by (auth_method) (hctv_chat_websocket_connections_by_auth_method)", - "legendFormat": "active {{auth_method}}", - "refId": "A" - }, - { - "expr": "sum by (auth_method, outcome) (rate(hctv_chat_websocket_connection_attempts_total[15m]))", - "legendFormat": "{{auth_method}} {{outcome}}", - "refId": "B" - } - ], - "title": "Socket Auth Mix", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true }, "unit": "ops" }, @@ -768,428 +879,22 @@ }, "gridPos": { "h": 8, - "w": 8, + "w": 12, "x": 0, - "y": 20 - }, - "id": 15, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "sum by (action) (rate(hctv_chat_moderation_actions_total[15m]))", - "legendFormat": "{{action}}", - "refId": "A" - }, - { - "expr": "sum by (reason) (rate(hctv_chat_moderation_blocks_total[15m]))", - "legendFormat": "blocked {{reason}}", - "refId": "B" - } - ], - "title": "Moderation Load", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 20 - }, - "id": 16, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "sum by (type) (rate(hctv_chat_inbound_payload_bytes_total[5m]))", - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "title": "Inbound Payload Rate by Type", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 20 - }, - "id": 17, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, type) (rate(hctv_chat_message_duration_seconds_bucket[15m])))", - "legendFormat": "{{type}} p95", - "refId": "A" - } - ], - "title": "Chat Processing Latency P95", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "reqps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 28 - }, - "id": 18, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "sum by (outcome) (rate(hctv_web_mediamtx_auth_requests_total[5m]))", - "legendFormat": "{{outcome}}", - "refId": "A" - } - ], - "title": "Media Auth Outcomes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 28 - }, - "id": 19, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, action, protocol) (rate(hctv_web_mediamtx_auth_duration_seconds_bucket[15m])))", - "legendFormat": "{{action}}/{{protocol}} p95", - "refId": "A" - } - ], - "title": "Media Auth Latency P95", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 28 - }, - "id": 20, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "sum by (job, status) (rate(hctv_web_background_job_runs_total[15m]))", - "legendFormat": "{{job}} {{status}}", - "refId": "A" - } - ], - "title": "Background Job Outcomes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 36 - }, - "id": 21, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum by (le, job) (rate(hctv_web_background_job_duration_seconds_bucket[15m])))", - "legendFormat": "{{job}} p95", - "refId": "A" - } - ], - "title": "Background Job Latency P95", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 36 - }, - "id": 22, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "sum by (target) (rate(hctv_web_notifications_enqueued_total[15m]))", - "legendFormat": "{{target}}", - "refId": "A" - } - ], - "title": "Notifications Enqueued", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 36 - }, - "id": 23, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "sum by (cache) (hctv_web_cache_entries)", - "legendFormat": "{{cache}}", - "refId": "A" - }, - { - "expr": "hctv_web_thumbnail_refresh_targets", - "legendFormat": "thumbnail targets", - "refId": "B" - } - ], - "title": "Cache + Thumbnail State", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 44 - }, - "id": 24, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "sum by (entity) (hctv_web_platform_inventory)", - "legendFormat": "{{entity}}", - "refId": "A" - } - ], - "title": "Platform Inventory", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "unit": "ops" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 44 + "y": 15 }, "id": 25, "options": { "legend": { "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "desc" } }, "targets": [ @@ -1217,35 +922,325 @@ "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 80, + "gradientMode": "none", + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 24, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (entity) (hctv_web_platform_inventory)", + "legendFormat": "{{entity}}", + "refId": "A" + } + ], + "title": "Platform Inventory", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 200, + "panels": [], + "title": "Chat", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "reqps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "bytes/s" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "Bps" + }, + { + "id": "custom.drawStyle", + "value": "line" + }, + { + "id": "custom.fillOpacity", + "value": 5 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 24 + }, + "id": 12, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum(rate(hctv_chat_messages_delivered_total[5m]))", + "legendFormat": "delivered msg/s", + "refId": "A" + }, + { + "expr": "sum(rate(hctv_chat_incoming_messages_total[5m]))", + "legendFormat": "inbound frames/s", + "refId": "B" + }, + { + "expr": "sum(rate(hctv_chat_message_bytes_delivered_total[5m]))", + "legendFormat": "bytes/s", + "refId": "C" + } + ], + "title": "Chat Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "normal" + } + }, "unit": "short" }, "overrides": [] }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 24 + }, + "id": 13, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (channel) (hctv_chat_websocket_connections_by_channel)", + "legendFormat": "{{channel}}", + "refId": "A" + } + ], + "title": "Socket Load by Channel", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "s" + }, + "overrides": [] + }, "gridPos": { "h": 8, "w": 8, "x": 16, - "y": 44 + "y": 24 + }, + "id": 17, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le, type) (rate(hctv_chat_message_duration_seconds_bucket[15m])))", + "legendFormat": "{{type}} p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, type) (rate(hctv_chat_message_duration_seconds_bucket[15m])))", + "legendFormat": "{{type}} p95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, sum by (le, type) (rate(hctv_chat_message_duration_seconds_bucket[15m])))", + "legendFormat": "{{type}} p99", + "refId": "C" + } + ], + "title": "Chat Processing Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 32 }, "id": 26, "options": { "legend": { "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "desc" } }, "targets": [ { "expr": "sum by (sender_type) (rate(hctv_chat_unique_chatters_total[1h]))", - "legendFormat": "{{sender_type}}", + "legendFormat": "chatters \u2014 {{sender_type}}", "refId": "A" }, { - "expr": "sum by (channel) (rate(hctv_chat_history_messages_loaded_total[15m]))", - "legendFormat": "history {{channel}}", + "expr": "sum by (sender_type) (rate(hctv_chat_messages_delivered_total[5m]))", + "legendFormat": "msg/s \u2014 {{sender_type}}", "refId": "B" } ], @@ -1262,34 +1257,49 @@ "color": { "mode": "palette-classic" }, - "unit": "short" + "custom": { + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, "w": 8, - "x": 0, - "y": 52 + "x": 8, + "y": 32 }, - "id": 27, + "id": 16, "options": { "legend": { "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "desc" } }, "targets": [ { - "expr": "sum by (channel, setting) (hctv_chat_moderation_state)", - "legendFormat": "{{channel}} {{setting}}", + "expr": "sum by (type) (rate(hctv_chat_inbound_payload_bytes_total[5m]))", + "legendFormat": "{{type}}", "refId": "A" } ], - "title": "Channel Moderation Settings", + "title": "Inbound Payload by Type", "type": "timeseries" }, { @@ -1302,45 +1312,14 @@ "color": { "mode": "palette-classic" }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 52 - }, - "id": 28, - "options": { - "legend": { - "displayMode": "table", - "placement": "bottom" - }, - "tooltip": { - "mode": "multi" - } - }, - "targets": [ - { - "expr": "sum by (channel) (hctv_chat_channel_history_size)", - "legendFormat": "{{channel}}", - "refId": "A" - } - ], - "title": "Chat History Footprint", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true }, "unit": "short" }, @@ -1350,31 +1329,114 @@ "h": 8, "w": 8, "x": 16, - "y": 52 + "y": 32 }, - "id": 29, + "id": 40, "options": { "legend": { "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "calcs": [ + "lastNotNull", + "mean" + ] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "desc" } }, "targets": [ { - "expr": "sum by (phase) (rate(hctv_chat_errors_total[15m]))", - "legendFormat": "chat {{phase}}", + "expr": "sum(rate(hctv_chat_emoji_search_results_count[5m]))", + "legendFormat": "searches/s", "refId": "A" }, { - "expr": "sum by (job, status) (rate(hctv_web_background_job_runs_total{status=\"error\"}[15m]))", - "legendFormat": "web {{job}} errors", + "expr": "histogram_quantile(0.50, sum by (le) (rate(hctv_chat_emoji_search_results_bucket{outcome=\"matched\"}[15m])))", + "legendFormat": "results p50", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.95, sum by (le) (rate(hctv_chat_emoji_search_results_bucket{outcome=\"matched\"}[15m])))", + "legendFormat": "results p95", + "refId": "C" + } + ], + "title": "Emoji Search Quality", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 40 + }, + "id": 300, + "panels": [], + "title": "Chat Moderation & Auth", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 41 + }, + "id": 15, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (action) (rate(hctv_chat_moderation_actions_total[15m]))", + "legendFormat": "{{action}}", + "refId": "A" + }, + { + "expr": "sum by (reason) (rate(hctv_chat_moderation_blocks_total[15m]))", + "legendFormat": "blocked: {{reason}}", "refId": "B" } ], - "title": "Application Errors", + "title": "Moderation Load", "type": "timeseries" }, { @@ -1387,6 +1449,15 @@ "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, "unit": "short" }, "overrides": [] @@ -1394,17 +1465,701 @@ "gridPos": { "h": 8, "w": 8, + "x": 8, + "y": 41 + }, + "id": 14, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (auth_method) (hctv_chat_websocket_connections_by_auth_method)", + "legendFormat": "active \u2014 {{auth_method}}", + "refId": "A" + }, + { + "expr": "sum by (auth_method, outcome) (rate(hctv_chat_websocket_connection_attempts_total[15m]))", + "legendFormat": "{{auth_method}} {{outcome}}/s", + "refId": "B" + } + ], + "title": "Socket Auth Mix", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 41 + }, + "id": 28, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (channel) (hctv_chat_channel_history_size)", + "legendFormat": "{{channel}}", + "refId": "A" + } + ], + "title": "Chat History Size by Channel", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 5, + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, "x": 0, - "y": 60 + "y": 49 + }, + "id": 27, + "options": { + "legend": { + "displayMode": "table", + "placement": "right", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (channel, setting) (hctv_chat_moderation_state)", + "legendFormat": "{{channel}} \u2014 {{setting}}", + "refId": "A" + } + ], + "title": "Channel Moderation Settings", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 57 + }, + "id": 400, + "panels": [], + "title": "Media Ingress & Auth", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 58 + }, + "id": 18, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (outcome) (rate(hctv_web_mediamtx_auth_requests_total[5m]))", + "legendFormat": "{{outcome}}", + "refId": "A" + } + ], + "title": "Media Auth Outcomes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 58 + }, + "id": 19, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le, action, protocol) (rate(hctv_web_mediamtx_auth_duration_seconds_bucket[15m])))", + "legendFormat": "{{action}}/{{protocol}} p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, action, protocol) (rate(hctv_web_mediamtx_auth_duration_seconds_bucket[15m])))", + "legendFormat": "{{action}}/{{protocol}} p95", + "refId": "B" + } + ], + "title": "Media Auth Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Native MediaMTX metrics: active HLS, SRT, and WebRTC connections", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 58 + }, + "id": 41, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "hls_muxers{job=\"mediamtx\"}", + "legendFormat": "HLS muxers", + "refId": "A" + }, + { + "expr": "srt_conns{job=\"mediamtx\"}", + "legendFormat": "SRT connections", + "refId": "B" + }, + { + "expr": "paths{job=\"mediamtx\"}", + "legendFormat": "active paths", + "refId": "C" + }, + { + "expr": "hls_muxers_bytes_sent{job=\"mediamtx\"}", + "legendFormat": "HLS bytes sent", + "refId": "D" + } + ], + "title": "MediaMTX Connections", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 66 + }, + "id": 500, + "panels": [], + "title": "Background Jobs & Caches", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 67 + }, + "id": 20, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (job, status) (rate(hctv_web_background_job_runs_total[15m]))", + "legendFormat": "{{job}} \u2014 {{status}}", + "refId": "A" + } + ], + "title": "Background Job Outcomes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 67 + }, + "id": 21, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "histogram_quantile(0.50, sum by (le, job) (rate(hctv_web_background_job_duration_seconds_bucket[15m])))", + "legendFormat": "{{job}} p50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, sum by (le, job) (rate(hctv_web_background_job_duration_seconds_bucket[15m])))", + "legendFormat": "{{job}} p95", + "refId": "B" + } + ], + "title": "Background Job Latency", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 67 + }, + "id": 22, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (target) (rate(hctv_web_notifications_enqueued_total[15m]))", + "legendFormat": "notify \u2014 {{target}}", + "refId": "A" + } + ], + "title": "Notifications Enqueued", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "lineInterpolation": "stepAfter", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 75 + }, + "id": 23, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (cache) (hctv_web_cache_entries)", + "legendFormat": "{{cache}}", + "refId": "A" + }, + { + "expr": "hctv_web_thumbnail_refresh_targets", + "legendFormat": "thumbnail targets", + "refId": "B" + } + ], + "title": "Cache & Thumbnail State", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 75 + }, + "id": 42, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (channel) (rate(hctv_chat_history_messages_loaded_total[15m]))", + "legendFormat": "history \u2014 {{channel}}", + "refId": "A" + } + ], + "title": "Chat History Loads by Channel", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 600, + "panels": [], + "title": "Infrastructure", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*commits/sec" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "ops" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 84 }, "id": 30, "options": { "legend": { "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "desc" } }, "targets": [ @@ -1427,12 +2182,41 @@ "type": "prometheus", "uid": "prometheus" }, + "description": "Cache hit ratio should stay above 99%. Lower values indicate the database is doing too many disk reads.", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "mode": "thresholds" }, - "unit": "bytes" + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 0.95 + }, + { + "color": "green", + "value": 0.99 + } + ] + }, + "min": 0, + "max": 1, + "unit": "percentunit" }, "overrides": [] }, @@ -1440,31 +2224,31 @@ "h": 8, "w": 8, "x": 8, - "y": 60 + "y": 84 }, - "id": 31, + "id": 43, "options": { "legend": { "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "calcs": [ + "lastNotNull", + "min" + ] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "desc" } }, "targets": [ { - "expr": "redis_memory_used_bytes", - "legendFormat": "redis memory", + "expr": "pg_stat_database_blks_hit{datname=\"hctv\"} / (pg_stat_database_blks_hit{datname=\"hctv\"} + pg_stat_database_blks_read{datname=\"hctv\"} + 1)", + "legendFormat": "cache hit ratio", "refId": "A" - }, - { - "expr": "redis_connected_clients", - "legendFormat": "connected clients", - "refId": "B" } ], - "title": "Redis Footprint", + "title": "Postgres Cache Hit Ratio", "type": "timeseries" }, { @@ -1477,6 +2261,257 @@ "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": ".*deadlocks.*" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "color", + "value": { + "mode": "fixed", + "fixedColor": "red" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 84 + }, + "id": 44, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "rate(pg_stat_database_xact_rollback{datname=\"hctv\"}[5m])", + "legendFormat": "rollbacks/s", + "refId": "A" + }, + { + "expr": "rate(pg_stat_database_deadlocks{datname=\"hctv\"}[5m])", + "legendFormat": "deadlocks/s", + "refId": "B" + }, + { + "expr": "rate(pg_stat_database_conflicts{datname=\"hctv\"}[5m])", + "legendFormat": "conflicts/s", + "refId": "C" + } + ], + "title": "Postgres Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "connected clients" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "short" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 92 + }, + "id": 31, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "redis_memory_used_bytes", + "legendFormat": "memory used", + "refId": "A" + }, + { + "expr": "redis_connected_clients", + "legendFormat": "connected clients", + "refId": "B" + } + ], + "title": "Redis Memory & Clients", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "ops" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "hit ratio" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "min", + "value": 0 + }, + { + "id": "max", + "value": 1 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 92 + }, + "id": 45, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "rate(redis_commands_processed_total[5m])", + "legendFormat": "commands/s", + "refId": "A" + }, + { + "expr": "redis_keyspace_hits_total / (redis_keyspace_hits_total + redis_keyspace_misses_total + 1)", + "legendFormat": "hit ratio", + "refId": "B" + } + ], + "title": "Redis Throughput", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, "unit": "bytes" }, "overrides": [] @@ -1485,43 +2520,318 @@ "h": 8, "w": 8, "x": 16, - "y": 60 + "y": 92 }, "id": 32, "options": { "legend": { "displayMode": "table", - "placement": "bottom" + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] }, "tooltip": { - "mode": "multi" + "mode": "multi", + "sort": "desc" } }, "targets": [ { "expr": "hctv_web_process_resident_memory_bytes", - "legendFormat": "web memory", + "legendFormat": "web RSS", "refId": "A" }, { "expr": "hctv_chat_process_resident_memory_bytes", - "legendFormat": "chat memory", + "legendFormat": "chat RSS", "refId": "B" }, { "expr": "process_resident_memory_bytes{job=\"mediamtx\"}", - "legendFormat": "mediamtx memory", + "legendFormat": "mediamtx RSS", "refId": "C" } ], "title": "Service Memory", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Node.js event loop lag from default metrics. Spikes indicate the main thread is blocked.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 100 + }, + "id": 46, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "hctv_web_nodejs_eventloop_lag_seconds", + "legendFormat": "web event loop lag", + "refId": "A" + }, + { + "expr": "hctv_chat_nodejs_eventloop_lag_seconds", + "legendFormat": "chat event loop lag", + "refId": "B" + } + ], + "title": "Node.js Event Loop Lag", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "description": "Active handles and file descriptors \u2014 watch for leaks over time.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "lineInterpolation": "smooth", + "lineWidth": 2, + "showPoints": "never", + "spanNulls": true + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 100 + }, + "id": 47, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull", + "max" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "hctv_web_nodejs_active_handles_total", + "legendFormat": "web active handles", + "refId": "A" + }, + { + "expr": "hctv_chat_nodejs_active_handles_total", + "legendFormat": "chat active handles", + "refId": "B" + }, + { + "expr": "hctv_web_process_open_fds", + "legendFormat": "web open FDs", + "refId": "C" + }, + { + "expr": "hctv_chat_process_open_fds", + "legendFormat": "chat open FDs", + "refId": "D" + } + ], + "title": "Process Handles & File Descriptors", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 108 + }, + "id": 700, + "panels": [], + "title": "Errors", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 109 + }, + "id": 29, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (phase) (rate(hctv_chat_errors_total[15m]))", + "legendFormat": "chat \u2014 {{phase}}", + "refId": "A" + }, + { + "expr": "sum by (job) (rate(hctv_web_background_job_runs_total{status=\"error\"}[15m]))", + "legendFormat": "web job \u2014 {{job}}", + "refId": "B" + } + ], + "title": "Application Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "bars", + "fillOpacity": 60, + "gradientMode": "none", + "lineWidth": 0, + "showPoints": "never", + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 109 + }, + "id": 48, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "calcs": [ + "lastNotNull" + ] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "expr": "sum by (outcome) (rate(hctv_web_mediamtx_auth_requests_total{outcome=~\"error|unauthorized\"}[15m]))", + "legendFormat": "media auth \u2014 {{outcome}}", + "refId": "A" + }, + { + "expr": "sum by (auth_method) (rate(hctv_chat_websocket_connection_attempts_total{outcome=\"rejected\"}[15m]))", + "legendFormat": "chat reject \u2014 {{auth_method}}", + "refId": "B" + }, + { + "expr": "sum by (region, status) (rate(hctv_web_stream_sync_scrapes_total{status=\"error\"}[15m]))", + "legendFormat": "sync scrape error \u2014 {{region}}", + "refId": "C" + } + ], + "title": "Auth & Sync Errors", + "type": "timeseries" } ], "refresh": "30s", "schemaVersion": 39, - "style": "dark", - "tags": ["hackclub.tv", "observability"], + "tags": [ + "hackclub.tv", + "observability" + ], "templating": { "list": [] }, @@ -1533,6 +2843,6 @@ "timezone": "browser", "title": "HackClub.tv Overview", "uid": "hctv-overview", - "version": 2, + "version": 4, "weekStart": "" }