{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": {
          "type": "grafana",
          "uid": "-- Grafana --"
        },
        "enable": true,
        "hide": true,
        "iconColor": "rgba(255, 211, 102, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "description": "Live health, streaming, chat, and infrastructure metrics for HackClub.tv",
  "editable": true,
  "fiscalYearStartMonth": 0,
  "graphTooltip": 1,
  "id": null,
  "links": [],
  "panels": [
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [
            {
              "options": {
                "0": {
                  "text": "DOWN"
                },
                "1": {
                  "text": "UP"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "#73BF69",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 4,
        "x": 0,
        "y": 0
      },
      "id": 1,
      "options": {
        "colorMode": "background_solid",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "expr": "max(up{job=\"web\"})",
          "refId": "A"
        }
      ],
      "title": "Web",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [
            {
              "options": {
                "0": {
                  "text": "DOWN"
                },
                "1": {
                  "text": "UP"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "#73BF69",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 4,
        "x": 4,
        "y": 0
      },
      "id": 2,
      "options": {
        "colorMode": "background_solid",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "expr": "max(up{job=\"chat\"})",
          "refId": "A"
        }
      ],
      "title": "Chat",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [
            {
              "options": {
                "0": {
                  "text": "DOWN"
                },
                "1": {
                  "text": "UP"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "#73BF69",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 4,
        "x": 8,
        "y": 0
      },
      "id": 3,
      "options": {
        "colorMode": "background_solid",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "expr": "max(up{job=\"mediamtx\"})",
          "refId": "A"
        }
      ],
      "title": "MediaMTX",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [
            {
              "options": {
                "0": {
                  "text": "DOWN"
                },
                "1": {
                  "text": "UP"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "#73BF69",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 4,
        "x": 12,
        "y": 0
      },
      "id": 4,
      "options": {
        "colorMode": "background_solid",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "expr": "max(pg_up)",
          "refId": "A"
        }
      ],
      "title": "Postgres",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "mappings": [
            {
              "options": {
                "0": {
                  "text": "DOWN"
                },
                "1": {
                  "text": "UP"
                }
              },
              "type": "value"
            }
          ],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "#73BF69",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 4,
        "x": 16,
        "y": 0
      },
      "id": 5,
      "options": {
        "colorMode": "background_solid",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "expr": "max(redis_up)",
          "refId": "A"
        }
      ],
      "title": "Redis",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              },
              {
                "color": "#FF9830",
                "value": 1
              },
              {
                "color": "#73BF69",
                "value": 5
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 4,
        "x": 20,
        "y": 0
      },
      "id": 6,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "expr": "sum(hctv_web_live_streams)",
          "refId": "A"
        }
      ],
      "title": "Live Streams",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              },
              {
                "color": "#73BF69",
                "value": 1
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 6,
        "x": 0,
        "y": 3
      },
      "id": 7,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "expr": "hctv_web_active_viewers",
          "refId": "A"
        }
      ],
      "title": "Viewers",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              },
              {
                "color": "#73BF69",
                "value": 1
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 6,
        "x": 6,
        "y": 3
      },
      "id": 8,
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "targets": [
        {
          "expr": "hctv_chat_websocket_connections",
          "refId": "A"
        }
      ],
      "title": "Chat Sockets",
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              },
              {
                "color": "#73BF69",
                "value": 1
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "type": "stat",
      "gridPos": {
        "h": 3,
        "w": 6,
        "x": 12,
        "y": 3
      },
      "id": 49,
      "targets": [
        {
          "expr": "hctv_web_platform_inventory{entity=\"channels\"}",
          "refId": "A"
        }
      ],
      "title": "Channels"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "text",
                "value": null
              },
              {
                "color": "#73BF69",
                "value": 1
              }
            ]
          },
          "unit": "short"
        },
        "overrides": []
      },
      "options": {
        "colorMode": "value",
        "graphMode": "area",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "value"
      },
      "type": "stat",
      "gridPos": {
        "h": 3,
        "w": 6,
        "x": 18,
        "y": 3
      },
      "id": 50,
      "targets": [
        {
          "expr": "hctv_web_platform_inventory{entity=\"users\"}",
          "refId": "A"
        }
      ],
      "title": "Onboarded Users"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 6
      },
      "id": 100,
      "panels": [],
      "title": "Streams & Viewers",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "axisCenteredZero": false,
            "axisLabel": "",
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 7
      },
      "id": 9,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (region) (hctv_web_live_streams)",
          "legendFormat": "live \u2014 {{region}}",
          "refId": "A"
        },
        {
          "expr": "sum by (region) (hctv_web_stream_paths_seen)",
          "legendFormat": "paths \u2014 {{region}}",
          "refId": "B"
        }
      ],
      "title": "Stream Estate by Region",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisBorderShow": false,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "pointSize": 5,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 7
      },
      "id": 10,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "hctv_web_active_viewers",
          "legendFormat": "total viewers",
          "refId": "A"
        },
        {
          "expr": "hctv_web_hottest_stream_viewers",
          "legendFormat": "top stream",
          "refId": "B"
        },
        {
          "expr": "hctv_web_streams_with_viewers",
          "legendFormat": "streams w/ viewers",
          "refId": "C"
        }
      ],
      "title": "Audience Pulse",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 20,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "normal"
            }
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 7
      },
      "id": 11,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (region) (hctv_web_active_viewers_by_region)",
          "legendFormat": "{{region}}",
          "refId": "A"
        }
      ],
      "title": "Viewers by Region",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "ops"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 15
      },
      "id": 25,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (region, transition) (rate(hctv_web_live_stream_transitions_total[15m]))",
          "legendFormat": "{{region}} {{transition}}",
          "refId": "A"
        },
        {
          "expr": "sum by (region, status) (rate(hctv_web_stream_sync_scrapes_total[15m]))",
          "legendFormat": "scrape {{region}} {{status}}",
          "refId": "B"
        }
      ],
      "title": "Stream Sync Health",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "bars",
            "fillOpacity": 80,
            "gradientMode": "none",
            "lineWidth": 0,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "normal"
            }
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 15
      },
      "id": 24,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (entity) (hctv_web_platform_inventory)",
          "legendFormat": "{{entity}}",
          "refId": "A"
        }
      ],
      "title": "Platform Inventory",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 23
      },
      "id": 200,
      "panels": [],
      "title": "Chat",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "reqps"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "bytes/s"
            },
            "properties": [
              {
                "id": "custom.axisPlacement",
                "value": "right"
              },
              {
                "id": "unit",
                "value": "Bps"
              },
              {
                "id": "custom.drawStyle",
                "value": "line"
              },
              {
                "id": "custom.fillOpacity",
                "value": 5
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 24
      },
      "id": 12,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum(rate(hctv_chat_messages_delivered_total[5m]))",
          "legendFormat": "delivered msg/s",
          "refId": "A"
        },
        {
          "expr": "sum(rate(hctv_chat_incoming_messages_total[5m]))",
          "legendFormat": "inbound frames/s",
          "refId": "B"
        },
        {
          "expr": "sum(rate(hctv_chat_message_bytes_delivered_total[5m]))",
          "legendFormat": "bytes/s",
          "refId": "C"
        }
      ],
      "title": "Chat Throughput",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 20,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true,
            "stacking": {
              "group": "A",
              "mode": "normal"
            }
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 24
      },
      "id": 13,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (channel) (hctv_chat_websocket_connections_by_channel)",
          "legendFormat": "{{channel}}",
          "refId": "A"
        }
      ],
      "title": "Socket Load by Channel",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 24
      },
      "id": 17,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.50, sum by (le, type) (rate(hctv_chat_message_duration_seconds_bucket[15m])))",
          "legendFormat": "{{type}} p50",
          "refId": "A"
        },
        {
          "expr": "histogram_quantile(0.95, sum by (le, type) (rate(hctv_chat_message_duration_seconds_bucket[15m])))",
          "legendFormat": "{{type}} p95",
          "refId": "B"
        },
        {
          "expr": "histogram_quantile(0.99, sum by (le, type) (rate(hctv_chat_message_duration_seconds_bucket[15m])))",
          "legendFormat": "{{type}} p99",
          "refId": "C"
        }
      ],
      "title": "Chat Processing Latency",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 32
      },
      "id": 26,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (sender_type) (rate(hctv_chat_unique_chatters_total[1h]))",
          "legendFormat": "chatters \u2014 {{sender_type}}",
          "refId": "A"
        },
        {
          "expr": "sum by (sender_type) (rate(hctv_chat_messages_delivered_total[5m]))",
          "legendFormat": "msg/s \u2014 {{sender_type}}",
          "refId": "B"
        }
      ],
      "title": "Chat Participation",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "bars",
            "fillOpacity": 60,
            "gradientMode": "none",
            "lineWidth": 0,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "normal"
            }
          },
          "unit": "Bps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 32
      },
      "id": 16,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (type) (rate(hctv_chat_inbound_payload_bytes_total[5m]))",
          "legendFormat": "{{type}}",
          "refId": "A"
        }
      ],
      "title": "Inbound Payload by Type",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 32
      },
      "id": 40,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull",
            "mean"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum(rate(hctv_chat_emoji_search_results_count[5m]))",
          "legendFormat": "searches/s",
          "refId": "A"
        },
        {
          "expr": "histogram_quantile(0.50, sum by (le) (rate(hctv_chat_emoji_search_results_bucket{outcome=\"matched\"}[15m])))",
          "legendFormat": "results p50",
          "refId": "B"
        },
        {
          "expr": "histogram_quantile(0.95, sum by (le) (rate(hctv_chat_emoji_search_results_bucket{outcome=\"matched\"}[15m])))",
          "legendFormat": "results p95",
          "refId": "C"
        }
      ],
      "title": "Emoji Search Quality",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 40
      },
      "id": 300,
      "panels": [],
      "title": "Chat Moderation & Auth",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "bars",
            "fillOpacity": 60,
            "gradientMode": "none",
            "lineWidth": 0,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "normal"
            }
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 41
      },
      "id": 15,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (action) (rate(hctv_chat_moderation_actions_total[15m]))",
          "legendFormat": "{{action}}",
          "refId": "A"
        },
        {
          "expr": "sum by (reason) (rate(hctv_chat_moderation_blocks_total[15m]))",
          "legendFormat": "blocked: {{reason}}",
          "refId": "B"
        }
      ],
      "title": "Moderation Load",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 41
      },
      "id": 14,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (auth_method) (hctv_chat_websocket_connections_by_auth_method)",
          "legendFormat": "active \u2014 {{auth_method}}",
          "refId": "A"
        },
        {
          "expr": "sum by (auth_method, outcome) (rate(hctv_chat_websocket_connection_attempts_total[15m]))",
          "legendFormat": "{{auth_method}} {{outcome}}/s",
          "refId": "B"
        }
      ],
      "title": "Socket Auth Mix",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 41
      },
      "id": 28,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (channel) (hctv_chat_channel_history_size)",
          "legendFormat": "{{channel}}",
          "refId": "A"
        }
      ],
      "title": "Chat History Size by Channel",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 5,
            "lineInterpolation": "stepAfter",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 24,
        "x": 0,
        "y": 49
      },
      "id": 27,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "right",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (channel, setting) (hctv_chat_moderation_state)",
          "legendFormat": "{{channel}} \u2014 {{setting}}",
          "refId": "A"
        }
      ],
      "title": "Channel Moderation Settings",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 57
      },
      "id": 400,
      "panels": [],
      "title": "Media Ingress & Auth",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "bars",
            "fillOpacity": 60,
            "gradientMode": "none",
            "lineWidth": 0,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "normal"
            }
          },
          "unit": "reqps"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 58
      },
      "id": 18,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (outcome) (rate(hctv_web_mediamtx_auth_requests_total[5m]))",
          "legendFormat": "{{outcome}}",
          "refId": "A"
        }
      ],
      "title": "Media Auth Outcomes",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 58
      },
      "id": 19,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.50, sum by (le, action, protocol) (rate(hctv_web_mediamtx_auth_duration_seconds_bucket[15m])))",
          "legendFormat": "{{action}}/{{protocol}} p50",
          "refId": "A"
        },
        {
          "expr": "histogram_quantile(0.95, sum by (le, action, protocol) (rate(hctv_web_mediamtx_auth_duration_seconds_bucket[15m])))",
          "legendFormat": "{{action}}/{{protocol}} p95",
          "refId": "B"
        }
      ],
      "title": "Media Auth Latency",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Native MediaMTX metrics: active HLS, SRT, and WebRTC connections",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 58
      },
      "id": 41,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "hls_muxers{job=\"mediamtx\"}",
          "legendFormat": "HLS muxers",
          "refId": "A"
        },
        {
          "expr": "srt_conns{job=\"mediamtx\"}",
          "legendFormat": "SRT connections",
          "refId": "B"
        },
        {
          "expr": "paths{job=\"mediamtx\"}",
          "legendFormat": "active paths",
          "refId": "C"
        },
        {
          "expr": "hls_muxers_bytes_sent{job=\"mediamtx\"}",
          "legendFormat": "HLS bytes sent",
          "refId": "D"
        }
      ],
      "title": "MediaMTX Connections",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 66
      },
      "id": 500,
      "panels": [],
      "title": "Background Jobs & Caches",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "bars",
            "fillOpacity": 60,
            "gradientMode": "none",
            "lineWidth": 0,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "normal"
            }
          },
          "unit": "ops"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 67
      },
      "id": 20,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (job, status) (rate(hctv_web_background_job_runs_total[15m]))",
          "legendFormat": "{{job}} \u2014 {{status}}",
          "refId": "A"
        }
      ],
      "title": "Background Job Outcomes",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 67
      },
      "id": 21,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "histogram_quantile(0.50, sum by (le, job) (rate(hctv_web_background_job_duration_seconds_bucket[15m])))",
          "legendFormat": "{{job}} p50",
          "refId": "A"
        },
        {
          "expr": "histogram_quantile(0.95, sum by (le, job) (rate(hctv_web_background_job_duration_seconds_bucket[15m])))",
          "legendFormat": "{{job}} p95",
          "refId": "B"
        }
      ],
      "title": "Background Job Latency",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 67
      },
      "id": 22,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (target) (rate(hctv_web_notifications_enqueued_total[15m]))",
          "legendFormat": "notify \u2014 {{target}}",
          "refId": "A"
        }
      ],
      "title": "Notifications Enqueued",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "none",
            "lineInterpolation": "stepAfter",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 75
      },
      "id": 23,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (cache) (hctv_web_cache_entries)",
          "legendFormat": "{{cache}}",
          "refId": "A"
        },
        {
          "expr": "hctv_web_thumbnail_refresh_targets",
          "legendFormat": "thumbnail targets",
          "refId": "B"
        }
      ],
      "title": "Cache & Thumbnail State",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 75
      },
      "id": 42,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (channel) (rate(hctv_chat_history_messages_loaded_total[15m]))",
          "legendFormat": "history \u2014 {{channel}}",
          "refId": "A"
        }
      ],
      "title": "Chat History Loads by Channel",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 83
      },
      "id": 600,
      "panels": [],
      "title": "Infrastructure",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": ".*commits/sec"
            },
            "properties": [
              {
                "id": "custom.axisPlacement",
                "value": "right"
              },
              {
                "id": "unit",
                "value": "ops"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 84
      },
      "id": 30,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "pg_stat_database_numbackends{datname=~\"hctv|postgres\"}",
          "legendFormat": "{{datname}} connections",
          "refId": "A"
        },
        {
          "expr": "rate(pg_stat_database_xact_commit{datname=~\"hctv|postgres\"}[5m])",
          "legendFormat": "{{datname}} commits/sec",
          "refId": "B"
        }
      ],
      "title": "Postgres Activity",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Cache hit ratio should stay above 99%. Lower values indicate the database is doing too many disk reads.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "red",
                "value": null
              },
              {
                "color": "yellow",
                "value": 0.95
              },
              {
                "color": "green",
                "value": 0.99
              }
            ]
          },
          "min": 0,
          "max": 1,
          "unit": "percentunit"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 84
      },
      "id": 43,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull",
            "min"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "pg_stat_database_blks_hit{datname=\"hctv\"} / (pg_stat_database_blks_hit{datname=\"hctv\"} + pg_stat_database_blks_read{datname=\"hctv\"} + 1)",
          "legendFormat": "cache hit ratio",
          "refId": "A"
        }
      ],
      "title": "Postgres Cache Hit Ratio",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byRegexp",
              "options": ".*deadlocks.*"
            },
            "properties": [
              {
                "id": "custom.axisPlacement",
                "value": "right"
              },
              {
                "id": "color",
                "value": {
                  "mode": "fixed",
                  "fixedColor": "red"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 84
      },
      "id": 44,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "rate(pg_stat_database_xact_rollback{datname=\"hctv\"}[5m])",
          "legendFormat": "rollbacks/s",
          "refId": "A"
        },
        {
          "expr": "rate(pg_stat_database_deadlocks{datname=\"hctv\"}[5m])",
          "legendFormat": "deadlocks/s",
          "refId": "B"
        },
        {
          "expr": "rate(pg_stat_database_conflicts{datname=\"hctv\"}[5m])",
          "legendFormat": "conflicts/s",
          "refId": "C"
        }
      ],
      "title": "Postgres Errors",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "bytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "connected clients"
            },
            "properties": [
              {
                "id": "custom.axisPlacement",
                "value": "right"
              },
              {
                "id": "unit",
                "value": "short"
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 0,
        "y": 92
      },
      "id": 31,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "redis_memory_used_bytes",
          "legendFormat": "memory used",
          "refId": "A"
        },
        {
          "expr": "redis_connected_clients",
          "legendFormat": "connected clients",
          "refId": "B"
        }
      ],
      "title": "Redis Memory & Clients",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "ops"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "hit ratio"
            },
            "properties": [
              {
                "id": "custom.axisPlacement",
                "value": "right"
              },
              {
                "id": "unit",
                "value": "percentunit"
              },
              {
                "id": "min",
                "value": 0
              },
              {
                "id": "max",
                "value": 1
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 8,
        "y": 92
      },
      "id": 45,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "rate(redis_commands_processed_total[5m])",
          "legendFormat": "commands/s",
          "refId": "A"
        },
        {
          "expr": "redis_keyspace_hits_total / (redis_keyspace_hits_total + redis_keyspace_misses_total + 1)",
          "legendFormat": "hit ratio",
          "refId": "B"
        }
      ],
      "title": "Redis Throughput",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 15,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 8,
        "x": 16,
        "y": 92
      },
      "id": 32,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "hctv_web_process_resident_memory_bytes",
          "legendFormat": "web RSS",
          "refId": "A"
        },
        {
          "expr": "hctv_chat_process_resident_memory_bytes",
          "legendFormat": "chat RSS",
          "refId": "B"
        },
        {
          "expr": "process_resident_memory_bytes{job=\"mediamtx\"}",
          "legendFormat": "mediamtx RSS",
          "refId": "C"
        }
      ],
      "title": "Service Memory",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Node.js event loop lag from default metrics. Spikes indicate the main thread is blocked.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "s"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 100
      },
      "id": 46,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "hctv_web_nodejs_eventloop_lag_seconds",
          "legendFormat": "web event loop lag",
          "refId": "A"
        },
        {
          "expr": "hctv_chat_nodejs_eventloop_lag_seconds",
          "legendFormat": "chat event loop lag",
          "refId": "B"
        }
      ],
      "title": "Node.js Event Loop Lag",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "description": "Active handles and file descriptors \u2014 watch for leaks over time.",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "lineInterpolation": "smooth",
            "lineWidth": 2,
            "showPoints": "never",
            "spanNulls": true
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 100
      },
      "id": 47,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull",
            "max"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "hctv_web_nodejs_active_handles_total",
          "legendFormat": "web active handles",
          "refId": "A"
        },
        {
          "expr": "hctv_chat_nodejs_active_handles_total",
          "legendFormat": "chat active handles",
          "refId": "B"
        },
        {
          "expr": "hctv_web_process_open_fds",
          "legendFormat": "web open FDs",
          "refId": "C"
        },
        {
          "expr": "hctv_chat_process_open_fds",
          "legendFormat": "chat open FDs",
          "refId": "D"
        }
      ],
      "title": "Process Handles & File Descriptors",
      "type": "timeseries"
    },
    {
      "collapsed": false,
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 108
      },
      "id": 700,
      "panels": [],
      "title": "Errors",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "bars",
            "fillOpacity": 60,
            "gradientMode": "none",
            "lineWidth": 0,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "normal"
            }
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 109
      },
      "id": 29,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (phase) (rate(hctv_chat_errors_total[15m]))",
          "legendFormat": "chat \u2014 {{phase}}",
          "refId": "A"
        },
        {
          "expr": "sum by (job) (rate(hctv_web_background_job_runs_total{status=\"error\"}[15m]))",
          "legendFormat": "web job \u2014 {{job}}",
          "refId": "B"
        }
      ],
      "title": "Application Errors",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "drawStyle": "bars",
            "fillOpacity": 60,
            "gradientMode": "none",
            "lineWidth": 0,
            "showPoints": "never",
            "stacking": {
              "group": "A",
              "mode": "normal"
            }
          },
          "unit": "short"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 109
      },
      "id": 48,
      "options": {
        "legend": {
          "displayMode": "table",
          "placement": "bottom",
          "calcs": [
            "lastNotNull"
          ]
        },
        "tooltip": {
          "mode": "multi",
          "sort": "desc"
        }
      },
      "targets": [
        {
          "expr": "sum by (outcome) (rate(hctv_web_mediamtx_auth_requests_total{outcome=~\"error|unauthorized\"}[15m]))",
          "legendFormat": "media auth \u2014 {{outcome}}",
          "refId": "A"
        },
        {
          "expr": "sum by (auth_method) (rate(hctv_chat_websocket_connection_attempts_total{outcome=\"rejected\"}[15m]))",
          "legendFormat": "chat reject \u2014 {{auth_method}}",
          "refId": "B"
        },
        {
          "expr": "sum by (region, status) (rate(hctv_web_stream_sync_scrapes_total{status=\"error\"}[15m]))",
          "legendFormat": "sync scrape error \u2014 {{region}}",
          "refId": "C"
        }
      ],
      "title": "Auth & Sync Errors",
      "type": "timeseries"
    }
  ],
  "refresh": "30s",
  "schemaVersion": 39,
  "tags": [
    "hackclub.tv",
    "observability"
  ],
  "templating": {
    "list": []
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "browser",
  "title": "HackClub.tv Overview",
  "uid": "hctv-overview",
  "version": 4,
  "weekStart": ""
}
