feat(observability): Grafana 看板激活生成指标 + 三条阈值告警(阶段四·面五之上)

- observability-overview.json:业务指标占位 panel 改用面五真实指标名(gen_task_total{status=succeeded} / gen_duration_seconds / gen_queue_depth / gen_gate_fail_total,去掉猜错的 huijing_ 前缀与 success 标签) + 新增 llm_cost 成本 panel(单次均值/近1h累计,标自估值非 new-api 权威)。 - provisioning/alerting/gen-alerts.yaml:三阈值告警——生成成功率<80%(MVP门 §2)/ 队列深度>15(便宜档并发上限)/ game-cloud up=0(后端不可达),+ webhook contact point(飞书/钉钉,URL 待创始人填)+ 路由策略。对着 MVP 门(成功率≥80% / 可用性≥99.5% / 便宜档并发≤15)。webhook URL 未填前告警在 Grafana UI 可见但不外发,§7 验收#2「告警真送达」待真 webhook。compose 已挂 provisioning 全目录,重启 grafana 容器即生效。 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
2026-07-05 20:47:52 -07:00 · 2026-07-05 20:47:52 -07:00 · 47644881ff
commit 47644881ff
parent 5b34807e73
2 changed files with 145 additions and 9 deletions
--- a/deploy/infra/observability/grafana/dashboards/observability-overview.json
+++ b/deploy/infra/observability/grafana/dashboards/observability-overview.json
@ -64,7 +64,7 @@
    },
    {
      "type": "row",
-      "title": "业务指标(占位 · 后续波埋点落地后生效)",
+      "title": "生成业务指标(面五 GenMetrics · 已生效 · 需 huijing.aigc.metrics.enabled=true)",
      "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 },
      "collapsed": false,
      "panels": []
@ -72,36 +72,48 @@
    {
      "type": "timeseries",
      "title": "生成成功率(按状态)",
-      "description": "占位:待 game-cloud 在任务终态回填处埋 huijing_gen_task_total{status}(设计 §6)。埋点前无数据。",
+      "description": "面五 GenMetrics:任务终态回填处埋 gen_task_total{status=succeeded/failed/timed_out}(§6)。成功率 = succeeded / 全部;需有终态任务才有数据。",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 },
      "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] },
      "targets": [
-        { "refId": "A", "expr": "sum(rate(huijing_gen_task_total{status=\"success\"}[30m])) / clamp_min(sum(rate(huijing_gen_task_total[30m])), 1)", "legendFormat": "成功率" }
+        { "refId": "A", "expr": "sum(rate(gen_task_total{status=\"succeeded\"}[30m])) / clamp_min(sum(rate(gen_task_total[30m])), 1)", "legendFormat": "成功率" }
      ]
    },
    {
      "type": "timeseries",
      "title": "生成耗时 P50/P95",
-      "description": "占位:待 worker 导出 huijing_gen_duration_seconds histogram(设计 §6)。埋点前无数据。",
+      "description": "面五 GenMetrics:gen_duration_seconds histogram(源 worker trace.wallS,§6)。便宜档需续修的 gen 延迟可达 20min,盯 P95。",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 },
      "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] },
      "targets": [
-        { "refId": "A", "expr": "histogram_quantile(0.50, sum(rate(huijing_gen_duration_seconds_bucket[5m])) by (le))", "legendFormat": "P50" },
-        { "refId": "B", "expr": "histogram_quantile(0.95, sum(rate(huijing_gen_duration_seconds_bucket[5m])) by (le))", "legendFormat": "P95" }
+        { "refId": "A", "expr": "histogram_quantile(0.50, sum(rate(gen_duration_seconds_bucket[5m])) by (le))", "legendFormat": "P50" },
+        { "refId": "B", "expr": "histogram_quantile(0.95, sum(rate(gen_duration_seconds_bucket[5m])) by (le))", "legendFormat": "P95" }
      ]
    },
    {
      "type": "timeseries",
      "title": "队列深度 / 九门失败",
-      "description": "占位:待埋 huijing_gen_queue_depth(gauge)与 huijing_gen_gate_fail_total{gate}(设计 §6)。埋点前无数据。",
+      "description": "面五 GenMetrics:gen_queue_depth(@Scheduled 刷在飞 queued+running 数)+ gen_gate_fail_total{gate}(九门逐门失败,§6)。",
      "datasource": { "type": "prometheus", "uid": "prometheus" },
      "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 },
      "fieldConfig": { "defaults": {}, "overrides": [] },
      "targets": [
-        { "refId": "A", "expr": "max(huijing_gen_queue_depth)", "legendFormat": "队列深度" },
-        { "refId": "B", "expr": "sum(rate(huijing_gen_gate_fail_total[30m])) by (gate)", "legendFormat": "门失败 {{gate}}" }
+        { "refId": "A", "expr": "max(gen_queue_depth)", "legendFormat": "队列深度" },
+        { "refId": "B", "expr": "sum(rate(gen_gate_fail_total[30m])) by (gate)", "legendFormat": "门失败 {{gate}}" }
+      ]
+    },
+    {
+      "type": "timeseries",
+      "title": "单次生成成本 ¥(自估值,非 new-api 权威计费)",
+      "description": "面五 GenMetrics:llm_cost DistributionSummary(源 worker trace.cost.totalRmb,§6/§7)。worker 按 token×倍率自估、非 new-api 权威流水,告警留余量。便宜档硬预算 < ¥10。",
+      "datasource": { "type": "prometheus", "uid": "prometheus" },
+      "gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 },
+      "fieldConfig": { "defaults": { "unit": "currencyCNY" }, "overrides": [] },
+      "targets": [
+        { "refId": "A", "expr": "sum(rate(llm_cost_sum[30m])) / clamp_min(sum(rate(llm_cost_count[30m])), 1)", "legendFormat": "单次均值¥" },
+        { "refId": "B", "expr": "sum(increase(llm_cost_sum[1h]))", "legendFormat": "近1h累计¥" }
      ]
    }
  ]
--- a/deploy/infra/observability/grafana/provisioning/alerting/gen-alerts.yaml
+++ b/deploy/infra/observability/grafana/provisioning/alerting/gen-alerts.yaml
@ -0,0 +1,124 @@
+# =============================================================================
+# Grafana 统一告警 provisioning —— 生成健康三条阈值告警(阶段四观测·面五指标之上)
+# 设计 §7/§10.1:告警件用 Grafana 统一告警(替下夜莺),自带规则 + 飞书/钉钉 webhook。
+#
+# ⚠️ webhook URL 待创始人填真实飞书/钉钉机器人地址(见 contactPoints.settings.url 的 TODO)。
+#    填前:告警规则仍会评估、触发、在 Grafana UI「Alerting」可见,只是不外发——
+#    §7 验收#2「告警真送达」需真 webhook 才能验;填后重启 grafana 容器即生效。
+#
+# 三规则对着 MVP 门:生成成功率 ≥80%(§2)、便宜档并发 ≤15、服务可用性 ≥99.5%。
+# 指标源 = 面五 GenMetrics(gen_task_total / gen_queue_depth)+ Prometheus up{job=game-cloud}。
+# =============================================================================
+apiVersion: 1
+
+groups:
+  - orgId: 1
+    name: 生成健康
+    folder: 绘境AI观测告警
+    interval: 1m
+    rules:
+      # ── 生成成功率跌破 80%(MVP 门 §2)──────────────────────────────────────
+      - uid: gen_success_rate_low
+        title: 生成成功率跌破80%(MVP门)
+        condition: C
+        for: 10m
+        noDataState: NoData
+        execErrState: Error
+        labels:
+          severity: warning
+          domain: gen
+        annotations:
+          summary: 便宜档生成成功率跌破 80%——查九门逐门失败(gen_gate_fail_total)与生成延迟(设计 §2 门 ≥80%)
+        data:
+          - refId: A
+            relativeTimeRange: { from: 1800, to: 0 }
+            datasourceUid: prometheus
+            model:
+              refId: A
+              instant: true
+              expr: sum(rate(gen_task_total{status="succeeded"}[30m])) / clamp_min(sum(rate(gen_task_total[30m])), 1)
+          - refId: C
+            relativeTimeRange: { from: 0, to: 0 }
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: lt, params: [0.8] }
+
+      # ── 在飞生成任务数超便宜档并发上限 15 ──────────────────────────────────
+      - uid: gen_queue_depth_high
+        title: 生成队列深度超上限15
+        condition: C
+        for: 5m
+        noDataState: NoData
+        execErrState: Error
+        labels:
+          severity: warning
+          domain: gen
+        annotations:
+          summary: 在飞生成任务数(gen_queue_depth)超便宜档并发上限 15,可能积压
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model:
+              refId: A
+              instant: true
+              expr: max(gen_queue_depth)
+          - refId: C
+            relativeTimeRange: { from: 0, to: 0 }
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: gt, params: [15] }
+
+      # ── game-cloud 后端不可达(可用性 ≥99.5% 门)─────────────────────────────
+      - uid: game_cloud_down
+        title: game-cloud后端不可达
+        condition: C
+        for: 2m
+        noDataState: Alerting
+        execErrState: Error
+        labels:
+          severity: critical
+          domain: infra
+        annotations:
+          summary: Prometheus 抓不到 game-cloud(100.64.0.7:48080/actuator/prometheus)up=0,后端可能挂了
+        data:
+          - refId: A
+            relativeTimeRange: { from: 300, to: 0 }
+            datasourceUid: prometheus
+            model:
+              refId: A
+              instant: true
+              expr: min(up{job="game-cloud"})
+          - refId: C
+            relativeTimeRange: { from: 0, to: 0 }
+            datasourceUid: __expr__
+            model:
+              refId: C
+              type: threshold
+              expression: A
+              conditions:
+                - evaluator: { type: lt, params: [1] }
+
+contactPoints:
+  - orgId: 1
+    name: ops-webhook
+    receivers:
+      - uid: ops_webhook_1
+        type: webhook
+        settings:
+          # TODO(创始人):填飞书/钉钉自定义机器人 webhook URL。填前告警在 Grafana UI 可见但不外发。
+          url: https://open.feishu.cn/open-apis/bot/v2/hook/REPLACE-WITH-REAL-TOKEN
+          httpMethod: POST
+
+policies:
+  - orgId: 1
+    receiver: ops-webhook
+    group_by: ["grafana_folder", "alertname"]