存储监控与告警

 ┌── Ceph 集群 ─────────────────────────────────────────────┐
 │ ceph-mgr (prometheus 模块)  :9283  健康/容量/PG/池        │
 │ ceph-exporter (每节点)      :9926  各守护进程 perf counter │
 │ node_exporter (每节点)      :9100  磁盘/网络/文件系统       │
 │ smartctl_exporter (每节点)  :9633  SMART、温度、磨损        │
 └───────────────┬──────────────────────────────────────────┘
                 │ scrape（或 telegraf 抓取后 remote write）
                 ▼
     Prometheus / VictoriaMetrics ──▶ Grafana（看板）
                 │
                 └──▶ 告警规则 ──▶ Alertmanager ──▶ IM / 电话 / 工单

ceph mgr module enable prometheus        # 已启用时无副作用
ceph mgr services                        # 看到 "prometheus": "http://192.168.10.11:9283/"
curl -s http://192.168.10.11:9283/metrics | grep -E '^ceph_health_status|^ceph_osd_up'

# 每个池的使用率（带池名）
ceph_pool_percent_used * on(pool_id) group_left(name) ceph_pool_metadata

# 最满的 5 个 OSD
topk(5, ceph_osd_stat_bytes_used / ceph_osd_stat_bytes)

# 每个 OSD 最近 5 分钟的平均读延迟（秒）
rate(ceph_osd_op_r_latency_sum[5m]) / rate(ceph_osd_op_r_latency_count[5m])

# 集群读 IOPS（写换成 ceph_osd_op_w）
sum(rate(ceph_osd_op_r[5m]))

# 不是 active 的 PG 数（> 0 意味着有数据不可访问）
sum(ceph_pg_total) - sum(ceph_pg_active)

radosgw-admin user create --uid extended-ceph-exporter --display-name "monitoring" \
  --caps "buckets=read;users=read;usage=read;metadata=read;zone=read"

# 利用率（等价 iostat 的 %util；对 NVMe 这类并行设备，100% 不代表饱和）
rate(node_disk_io_time_seconds_total{device=~"nvme.*|sd.*"}[5m])

# 读平均延迟 r_await（秒）
rate(node_disk_read_time_seconds_total[5m]) / rate(node_disk_reads_completed_total[5m])

# 平均队列深度（等价 aqu-sz）
rate(node_disk_io_time_weighted_seconds_total[5m])

# 文件系统剩余空间与 inode（MON 的数据盘、日志盘满了同样会出事）
node_filesystem_avail_bytes{mountpoint="/var/lib/ceph"} / node_filesystem_size_bytes{mountpoint="/var/lib/ceph"}
node_filesystem_files_free

scrape_configs:
  - job_name: ceph
    honor_labels: true
    static_configs:
      - targets: ["192.168.10.11:9283", "192.168.10.12:9283", "192.168.10.13:9283"]
  - job_name: ceph-exporter
    honor_labels: true
    static_configs:
      - targets: ["192.168.10.11:9926", "192.168.10.12:9926", "192.168.10.13:9926"]
  - job_name: node
    static_configs:
      - targets: ["192.168.10.11:9100", "192.168.10.12:9100", "192.168.10.13:9100"]
  - job_name: smartctl
    scrape_interval: 60s          # SMART 数据变化慢，没必要频繁读盘
    static_configs:
      - targets: ["192.168.10.11:9633", "192.168.10.12:9633", "192.168.10.13:9633"]

[agent]
  interval = "15s"
  flush_interval = "30s"

[[inputs.prometheus]]
  urls = ["http://192.168.10.11:9283/metrics", "http://192.168.10.12:9283/metrics"]

[[outputs.http]]
  url = "https://metrics.example.com/api/v1/write"
  data_format = "prometheusremotewrite"
  [outputs.http.headers]
    Content-Type = "application/x-protobuf"
    Content-Encoding = "snappy"
    X-Prometheus-Remote-Write-Version = "0.1.0"

vmsingle:
  spec:
    retentionPeriod: "30d"
    storage:
      resources: { requests: { storage: 1Ti } }
victoria-metrics-operator:
  operator:
    disable_prometheus_converter: false   # 转换 ServiceMonitor / PrometheusRule

apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  name: ceph-alerts
  namespace: monitoring
spec:
  groups:
    - name: ceph-health
      rules:
        - alert: CephHealthError
          expr: ceph_health_status == 2
          for: 5m
          labels: { severity: critical }
          annotations:
            summary: "Ceph 集群 HEALTH_ERR"
            description: "执行 ceph health detail 查看原因"
            runbook_url: "https://wiki.example.com/runbooks/ceph-health-error"
        - alert: CephHealthWarning
          expr: ceph_health_status == 1
          for: 15m
          labels: { severity: warning }
          annotations:
            summary: "Ceph 集群 HEALTH_WARN 超过 15 分钟"
        - alert: CephPGsInactive
          expr: sum(ceph_pg_total) - sum(ceph_pg_active) > 0
          for: 5m
          labels: { severity: critical }
          annotations:
            summary: "{{ $value }} 个 PG 不是 active，部分数据无法读写"
        - alert: CephPGInconsistent
          expr: sum(ceph_pg_inconsistent) > 0
          for: 5m
          labels: { severity: warning }
          annotations:
            summary: "scrub 发现 {{ $value }} 个不一致的 PG"
        - alert: CephMonOutOfQuorum
          expr: count(ceph_mon_quorum_status == 1) < count(ceph_mon_quorum_status)
          for: 5m
          labels: { severity: warning }
          annotations:
            summary: "有 MON 不在仲裁中，再坏一个可能失去多数派"
        - alert: CephOSDDown
          expr: count(ceph_osd_up == 0) > 0
          for: 5m
          labels: { severity: warning }
          annotations:
            summary: "{{ $value }} 个 OSD down"
        - alert: CephSlowOps
          expr: ceph_healthcheck_slow_ops > 0
          for: 2m
          labels: { severity: warning }
          annotations:
            summary: "存在 {{ $value }} 个 slow ops，客户端 I/O 可能卡住"
        - alert: CephNooutLeftOn
          expr: ceph_osd_flag_noout == 1
          for: 2h
          labels: { severity: warning }
          annotations:
            summary: "noout 已设置超过 2 小时，维护结束后是否忘了取消？"

    - name: ceph-capacity
      rules:
        - alert: CephOSDNearFull
          # 早于 Ceph 的 nearfull(0.85) 告警，留出处理时间
          expr: ceph_osd_stat_bytes_used / ceph_osd_stat_bytes > 0.80
          for: 10m
          labels: { severity: warning }
          annotations:
            summary: "{{ $labels.ceph_daemon }} 使用率超过 80%"
        - alert: CephPoolWillFillIn30d
          expr: |
            predict_linear(ceph_pool_stored[7d], 30 * 86400)
              > ceph_pool_stored + ceph_pool_max_avail
          for: 1h
          labels: { severity: warning }
          annotations:
            summary: "池 {{ $labels.pool_id }} 按近 7 天的增速，30 天内写满"

    - name: ceph-performance
      rules:
        - alert: CephOSDHighReadLatency
          expr: |
            rate(ceph_osd_op_r_latency_sum[5m]) / rate(ceph_osd_op_r_latency_count[5m]) > 0.05
          for: 10m
          labels: { severity: warning }
          annotations:
            summary: "{{ $labels.ceph_daemon }} 平均读延迟超过 50ms，检查对应磁盘"

    - name: disk-hardware
      rules:
        - alert: DiskSmartFailed
          expr: smartctl_device_smart_status == 0
          labels: { severity: critical }
          annotations:
            summary: "{{ $labels.instance }} {{ $labels.device }} SMART 自检失败，准备换盘"
        - alert: NVMeMediaErrorsIncreasing
          expr: increase(smartctl_device_media_errors[1h]) > 0
          labels: { severity: warning }
          annotations:
            summary: "{{ $labels.device }} 最近 1 小时出现新的介质错误"
        - alert: NVMeWearHigh
          expr: smartctl_device_percentage_used > 90
          labels: { severity: warning }
          annotations:
            summary: "{{ $labels.device }} 寿命已消耗 {{ $value }}%"

# 集群 raw 使用量 30 天后是否超过 85%（nearfull 线）
predict_linear(ceph_cluster_total_used_raw_bytes[7d], 30 * 86400)
  > ceph_cluster_total_bytes * 0.85

# 每个池还有多少天写满（结果单位：天），适合放进看板
ceph_pool_max_avail / clamp_min(deriv(ceph_pool_stored[7d]), 1) / 86400

route:
  receiver: im-storage
  group_by: [alertname, cluster]   # 同一类告警合并成一条通知
  group_wait: 30s
  group_interval: 5m
  repeat_interval: 4h              # 没恢复的告警 4 小时提醒一次，而不是每 5 分钟
  routes:
    - matchers: [severity="critical"]
      receiver: phone-oncall
      repeat_interval: 1h

inhibit_rules:
  # 整台节点挂了，就不要再为上面的每个 OSD 单独告警
  - source_matchers: [alertname="NodeDown"]
    target_matchers: [alertname=~"CephOSDDown|DiskSmartFailed"]
    equal: [instance]
  # 已经 HEALTH_ERR 了，WARN 就不用再发
  - source_matchers: [alertname="CephHealthError"]
    target_matchers: [alertname="CephHealthWarning"]
    equal: [cluster]