diff --git a/deploy/infra/observability/grafana/dashboards/observability-overview.json b/deploy/infra/observability/grafana/dashboards/observability-overview.json index d38d8260..a151cec4 100644 --- a/deploy/infra/observability/grafana/dashboards/observability-overview.json +++ b/deploy/infra/observability/grafana/dashboards/observability-overview.json @@ -64,7 +64,7 @@ }, { "type": "row", - "title": "业务指标(占位 · 后续波埋点落地后生效)", + "title": "生成业务指标(面五 GenMetrics · 已生效 · 需 huijing.aigc.metrics.enabled=true)", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 15 }, "collapsed": false, "panels": [] @@ -72,36 +72,48 @@ { "type": "timeseries", "title": "生成成功率(按状态)", - "description": "占位:待 game-cloud 在任务终态回填处埋 huijing_gen_task_total{status}(设计 §6)。埋点前无数据。", + "description": "面五 GenMetrics:任务终态回填处埋 gen_task_total{status=succeeded/failed/timed_out}(§6)。成功率 = succeeded / 全部;需有终态任务才有数据。", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 }, "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] }, "targets": [ - { "refId": "A", "expr": "sum(rate(huijing_gen_task_total{status=\"success\"}[30m])) / clamp_min(sum(rate(huijing_gen_task_total[30m])), 1)", "legendFormat": "成功率" } + { "refId": "A", "expr": "sum(rate(gen_task_total{status=\"succeeded\"}[30m])) / clamp_min(sum(rate(gen_task_total[30m])), 1)", "legendFormat": "成功率" } ] }, { "type": "timeseries", "title": "生成耗时 P50/P95", - "description": "占位:待 worker 导出 huijing_gen_duration_seconds histogram(设计 §6)。埋点前无数据。", + "description": "面五 GenMetrics:gen_duration_seconds histogram(源 worker trace.wallS,§6)。便宜档需续修的 gen 延迟可达 20min,盯 P95。", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 }, "fieldConfig": { "defaults": { "unit": "s" }, "overrides": [] }, "targets": [ - { "refId": "A", "expr": "histogram_quantile(0.50, sum(rate(huijing_gen_duration_seconds_bucket[5m])) by (le))", "legendFormat": "P50" }, - { "refId": "B", "expr": "histogram_quantile(0.95, sum(rate(huijing_gen_duration_seconds_bucket[5m])) by (le))", "legendFormat": "P95" } + { "refId": "A", "expr": "histogram_quantile(0.50, sum(rate(gen_duration_seconds_bucket[5m])) by (le))", "legendFormat": "P50" }, + { "refId": "B", "expr": "histogram_quantile(0.95, sum(rate(gen_duration_seconds_bucket[5m])) by (le))", "legendFormat": "P95" } ] }, { "type": "timeseries", "title": "队列深度 / 九门失败", - "description": "占位:待埋 huijing_gen_queue_depth(gauge)与 huijing_gen_gate_fail_total{gate}(设计 §6)。埋点前无数据。", + "description": "面五 GenMetrics:gen_queue_depth(@Scheduled 刷在飞 queued+running 数)+ gen_gate_fail_total{gate}(九门逐门失败,§6)。", "datasource": { "type": "prometheus", "uid": "prometheus" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 }, "fieldConfig": { "defaults": {}, "overrides": [] }, "targets": [ - { "refId": "A", "expr": "max(huijing_gen_queue_depth)", "legendFormat": "队列深度" }, - { "refId": "B", "expr": "sum(rate(huijing_gen_gate_fail_total[30m])) by (gate)", "legendFormat": "门失败 {{gate}}" } + { "refId": "A", "expr": "max(gen_queue_depth)", "legendFormat": "队列深度" }, + { "refId": "B", "expr": "sum(rate(gen_gate_fail_total[30m])) by (gate)", "legendFormat": "门失败 {{gate}}" } + ] + }, + { + "type": "timeseries", + "title": "单次生成成本 ¥(自估值,非 new-api 权威计费)", + "description": "面五 GenMetrics:llm_cost DistributionSummary(源 worker trace.cost.totalRmb,§6/§7)。worker 按 token×倍率自估、非 new-api 权威流水,告警留余量。便宜档硬预算 < ¥10。", + "datasource": { "type": "prometheus", "uid": "prometheus" }, + "gridPos": { "h": 8, "w": 8, "x": 0, "y": 24 }, + "fieldConfig": { "defaults": { "unit": "currencyCNY" }, "overrides": [] }, + "targets": [ + { "refId": "A", "expr": "sum(rate(llm_cost_sum[30m])) / clamp_min(sum(rate(llm_cost_count[30m])), 1)", "legendFormat": "单次均值¥" }, + { "refId": "B", "expr": "sum(increase(llm_cost_sum[1h]))", "legendFormat": "近1h累计¥" } ] } ] diff --git a/deploy/infra/observability/grafana/provisioning/alerting/gen-alerts.yaml b/deploy/infra/observability/grafana/provisioning/alerting/gen-alerts.yaml new file mode 100644 index 00000000..95cdc494 --- /dev/null +++ b/deploy/infra/observability/grafana/provisioning/alerting/gen-alerts.yaml @@ -0,0 +1,124 @@ +# ============================================================================= +# Grafana 统一告警 provisioning —— 生成健康三条阈值告警(阶段四观测·面五指标之上) +# 设计 §7/§10.1:告警件用 Grafana 统一告警(替下夜莺),自带规则 + 飞书/钉钉 webhook。 +# +# ⚠️ webhook URL 待创始人填真实飞书/钉钉机器人地址(见 contactPoints.settings.url 的 TODO)。 +# 填前:告警规则仍会评估、触发、在 Grafana UI「Alerting」可见,只是不外发—— +# §7 验收#2「告警真送达」需真 webhook 才能验;填后重启 grafana 容器即生效。 +# +# 三规则对着 MVP 门:生成成功率 ≥80%(§2)、便宜档并发 ≤15、服务可用性 ≥99.5%。 +# 指标源 = 面五 GenMetrics(gen_task_total / gen_queue_depth)+ Prometheus up{job=game-cloud}。 +# ============================================================================= +apiVersion: 1 + +groups: + - orgId: 1 + name: 生成健康 + folder: 绘境AI观测告警 + interval: 1m + rules: + # ── 生成成功率跌破 80%(MVP 门 §2)────────────────────────────────────── + - uid: gen_success_rate_low + title: 生成成功率跌破80%(MVP门) + condition: C + for: 10m + noDataState: NoData + execErrState: Error + labels: + severity: warning + domain: gen + annotations: + summary: 便宜档生成成功率跌破 80%——查九门逐门失败(gen_gate_fail_total)与生成延迟(设计 §2 门 ≥80%) + data: + - refId: A + relativeTimeRange: { from: 1800, to: 0 } + datasourceUid: prometheus + model: + refId: A + instant: true + expr: sum(rate(gen_task_total{status="succeeded"}[30m])) / clamp_min(sum(rate(gen_task_total[30m])), 1) + - refId: C + relativeTimeRange: { from: 0, to: 0 } + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: lt, params: [0.8] } + + # ── 在飞生成任务数超便宜档并发上限 15 ────────────────────────────────── + - uid: gen_queue_depth_high + title: 生成队列深度超上限15 + condition: C + for: 5m + noDataState: NoData + execErrState: Error + labels: + severity: warning + domain: gen + annotations: + summary: 在飞生成任务数(gen_queue_depth)超便宜档并发上限 15,可能积压 + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: + refId: A + instant: true + expr: max(gen_queue_depth) + - refId: C + relativeTimeRange: { from: 0, to: 0 } + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: gt, params: [15] } + + # ── game-cloud 后端不可达(可用性 ≥99.5% 门)───────────────────────────── + - uid: game_cloud_down + title: game-cloud后端不可达 + condition: C + for: 2m + noDataState: Alerting + execErrState: Error + labels: + severity: critical + domain: infra + annotations: + summary: Prometheus 抓不到 game-cloud(100.64.0.7:48080/actuator/prometheus)up=0,后端可能挂了 + data: + - refId: A + relativeTimeRange: { from: 300, to: 0 } + datasourceUid: prometheus + model: + refId: A + instant: true + expr: min(up{job="game-cloud"}) + - refId: C + relativeTimeRange: { from: 0, to: 0 } + datasourceUid: __expr__ + model: + refId: C + type: threshold + expression: A + conditions: + - evaluator: { type: lt, params: [1] } + +contactPoints: + - orgId: 1 + name: ops-webhook + receivers: + - uid: ops_webhook_1 + type: webhook + settings: + # TODO(创始人):填飞书/钉钉自定义机器人 webhook URL。填前告警在 Grafana UI 可见但不外发。 + url: https://open.feishu.cn/open-apis/bot/v2/hook/REPLACE-WITH-REAL-TOKEN + httpMethod: POST + +policies: + - orgId: 1 + receiver: ops-webhook + group_by: ["grafana_folder", "alertname"]