# 查询过去 30 天触发最多的告警 Top 20
topk(20,
  sum by (alertname) (
    rate(ALERTS{alertstate="firing"}[30d])
  )
)
# 按触发次数排序，优先处理 Top 20

-- 查找过去 30 天触发超过 100 次但从未导致客户影响的告警
SELECT alertname, COUNT(*) AS fires
FROM prometheus_alerts
WHERE fired_at > now() - INTERVAL '30 days'
GROUP BY alertname
HAVING COUNT(*) > 100
ORDER BY fires DESC;

# ❌ 删掉这种告警
- alert: HighCPU
  expr: 100 - (avg by(instance)(rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
  for: 1m
  labels:
    severity: warning

# ✅ CPU 使用率是指标，不是告警
# 如果 CPU 高导致请求延迟增加，告警延迟而不是 CPU
# 如果 CPU 高导致 OOM，告警 OOM 事件而不是 CPU

# ❌ 删掉——Pod 重启是 K8s 自愈机制，不是故障
- alert: PodRestarted
  expr: increase(kube_pod_container_status_restarts_total[1h]) > 0

# ✅ 只告 CrashLoopBackOff——持续重启才是问题
- alert: PodCrashLooping
  expr: increase(kube_pod_container_status_restarts_total[1h]) > 5
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: "Pod {{ $labels.pod }} 在过去1小时内重启超过5次"
    runbook_url: "https://runbook.example.com/crashloop"

# ❌ 删掉——HPA 扩缩容是设计行为，不是故障
- alert: HPA scaled
  expr: changes(kube_hpa_status_current_replicas[1h]) > 0

# ✅ 只告 HPA 无法扩容——真正的问题
- alert: HPAScalingLimited
  expr: kube_hpa_status_condition{condition="ScalingLimited", status="true"}
  for: 15m
  labels:
    severity: warning

# ❌ 删掉——阈值太低，频繁误报
- alert: DiskSpaceLow
  expr: 100 - (node_filesystem_avail_bytes / node_filesystem_size_bytes * 100) > 70
  for: 1m

# ✅ 用预测告警——给运维预留处理时间
- alert: DiskWillFillIn24h
  expr: predict_linear(node_filesystem_avail_bytes[1h], 24 * 3600) < 0
  for: 10m
  labels:
    severity: warning
  annotations:
    summary: "磁盘 {{ $labels.device }} 将在24小时内写满"

# payment-api 的 SLO 定义：可用性 99.9%
# 错误预算 = 1 - 99.9% = 0.1%
# 月预算 = 43200 秒 × 0.1% = 43.2 秒（允许宕机时间）

# Page 级告警：2% 预算在 1 小时内烧完（快燃烧）
- alert: PaymentApiSLOBurnRateFast
  expr: |
    (
      sum(rate(http_requests_total{job="payment-api", status=~"5.."}[1h]))
      /
      sum(rate(http_requests_total{job="payment-api"}[1h]))
    ) > (14.4 * 0.001)
    and
    (
      sum(rate(http_requests_total{job="payment-api", status=~"5.."}[5m]))
      /
      sum(rate(http_requests_total{job="payment-api"}[5m]))
    ) > (14.4 * 0.001)
  for: 2m
  labels:
    severity: page
    slo: payment-api-availability
  annotations:
    summary: "payment-api 错误预算快速燃烧"
    description: "2% 的月错误预算在1小时内消耗完毕，按当前速度3小时内预算耗尽"
    runbook_url: "https://runbook.example.com/payment-slo"

# Ticket 级告警：10% 预算在 3 天内烧完（慢燃烧）
- alert: PaymentApiSLOBurnRateSlow
  expr: |
    (
      sum(rate(http_requests_total{job="payment-api", status=~"5.."}[3d]))
      /
      sum(rate(http_requests_total{job="payment-api"}[3d]))
    ) > (1 * 0.001)
    and
    (
      sum(rate(http_requests_total{job="payment-api", status=~"5.."}[6h]))
      /
      sum(rate(http_requests_total{job="payment-api"}[6h]))
    ) > (1 * 0.001)
  for: 15m
  labels:
    severity: ticket
    slo: payment-api-availability
  annotations:
    summary: "payment-api 错误预算慢性消耗"
    description: "10% 的月错误预算在3天内消耗完毕，需要排查慢性问题"
    runbook_url: "https://runbook.example.com/payment-slo"

# alertmanager.yml
route:
  group_by: ['alertname', 'cluster', 'service']
  group_wait: 30s        # 首次告警等待 30s，攒齐同一组的告警
  group_interval: 5m     # 同组新告警的合并间隔
  repeat_interval: 4h    # 重复提醒间隔
  receiver: 'oncall-slack'

  routes:
    # Critical 级别：立即通知，短间隔重发
    - matchers:
        - severity="critical"
      group_wait: 10s
      group_interval: 1m
      repeat_interval: 1h
      receiver: 'oncall-pagerduty'

    # Warning 级别：发 Slack，工作时间通知
    - matchers:
        - severity="warning"
      group_wait: 30s
      group_interval: 5m
      repeat_interval: 4h
      receiver: 'oncall-slack'

    # Info 级别：只发邮件日报
    - matchers:
        - severity="info"
      group_wait: 5m
      group_interval: 1h
      repeat_interval: 24h
      receiver: 'daily-email'

# alertmanager.yml — 抑制规则
inhibit_rules:
  # 规则1：数据库挂了，抑制所有依赖该数据库的服务告警
  - source_matchers:
      - alertname="DatabaseDown"
    target_matchers:
      - alertname="ServiceHighErrorRate"
    equal: ['cluster']

  # 规则2：节点挂了，抑制该节点上所有 Pod 的告警
  - source_matchers:
      - alertname="NodeDown"
    target_matchers:
      - alertname=~"PodCrashLooping|PodNotReady|ContainerOOMKilled"
    equal: ['node']

  # 规则3：Critical 告警抑制同服务的 Warning 告警
  - source_matchers:
      - severity="critical"
    target_matchers:
      - severity="warning"
    equal: ['alertname', 'cluster', 'service']

  # 规则4：集群不可达，抑制所有该集群的告警
  - source_matchers:
      - alertname="ClusterUnreachable"
    target_matchers:
      - cluster="{{ $labels.cluster }}"
    equal: ['cluster']

# 发布系统在滚动更新前调用 Alertmanager API 创建静默
curl -XPOST http://alertmanager:9093/api/v2/silences \
  -H 'Content-Type: application/json' \
  -d '{
    "matchers": [
      {
        "name": "service",
        "value": "payment-api",
        "isRegex": false
      },
      {
        "name": "alertname",
        "value": "PodNotReady|ContainerOOMKilled|ProbeFailed",
        "isRegex": true
      }
    ],
    "startsAt": "2026-07-27T10:00:00Z",
    "endsAt": "2026-07-27T10:15:00Z",
    "createdBy": "deploy-bot",
    "comment": "payment-api 滚动发布，预计15分钟"
  }'

# 发布完成后，静默规则自动过期（endsAt 到期）
# 如果发布提前完成，主动删除静默规则
curl -XDELETE http://alertmanager:9093/api/v2/silence/{silence_id}

# alertmanager.yml — 按团队路由
route:
  receiver: 'default-slack'
  routes:
    # 支付团队
    - matchers:
        - team="payment"
      receiver: 'payment-slack'
      routes:
        - matchers:
            - severity="critical"
          receiver: 'payment-pagerduty'

    # 基础设施团队
    - matchers:
        - team="infra"
      receiver: 'infra-slack'
      routes:
        - matchers:
            - severity="critical"
          receiver: 'infra-pagerduty'

    # 业务团队
    - matchers:
        - team="business"
      receiver: 'business-slack'

# 每条告警必须包含以下 annotations
- alert: PaymentApiHighErrorRate
  expr: |
    sum(rate(http_requests_total{job="payment-api", status=~"5.."}[5m]))
    / sum(rate(http_requests_total{job="payment-api"}[5m])) > 0.01
  for: 5m
  labels:
    severity: page
    team: payment
    service: payment-api
  annotations:
    summary: "payment-api 5xx 错误率超过 1%"
    description: "当前错误率 {{ $value | humanizePercentage }}，持续 5 分钟"
    runbook_url: "https://runbook.example.com/payment-5xx"
    dashboard_url: "https://grafana.example.com/d/payment-api"
    quick_check: "kubectl logs -l app=payment-api --tail=50 | grep -i error"
    escalate_to: "#payment-oncall"

# scripts/validate-alerts.py — CI 检查脚本
import yaml, sys

def validate_alerts(rules_file):
with open(rules_file) as f:
        rules = yaml.safe_load(f)

    errors = []
    required_labels = ['severity', 'team', 'service']
    required_annotations = ['summary', 'runbook_url', 'dashboard_url']

for group in rules.get('groups', []):
for rule in group.get('rules', []):
if'alert'notin rule:
continue

            name = rule['alert']

# 检查必需 labels
for label in required_labels:
if label notin rule.get('labels', {}):
                    errors.append(f"Alert '{name}' missing label: {label}")

# 检查必需 annotations
for ann in required_annotations:
if ann notin rule.get('annotations', {}):
                    errors.append(f"Alert '{name}' missing annotation: {ann}")

# 检查 for 持续时间
if'for'notin rule:
                errors.append(f"Alert '{name}' missing 'for' duration")

if errors:
for e in errors:
            print(f"  X {e}")
        sys.exit(1)
else:
        print("  All alerts validated")

validate_alerts(sys.argv[1])

# .gitlab-ci.yml 或 GitHub Actions
validate-alerts:
  stage: test
  script:
    - python scripts/validate-alerts.py alerts/rules.yaml
  # 不通过则阻止合入

# 查看当前所有 firing 告警
amtool --alertmanager.url=http://alertmanager:9093 alert query

# 测试告警路由（不实际发送）
amtool --alertmanager.url=http://alertmanager:9093 config routes test \
  --severity=critical --team=payment --service=payment-api

# 创建静默规则
amtool --alertmanager.url=http://alertmanager:9093 silence add \
  alertname=PodCrashLooping service=payment-api \
  --duration=15m \
  --comment="发布期间静默" \
  --author="deploy-bot"

# 查看所有静默规则
amtool --alertmanager.url=http://alertmanager:9093 silence query

# 删除静默规则
amtool --alertmanager.url=http://alertmanager:9093 silence expire <silence_id>