Ceph 故障排查闯关

 ceph -s                     ← 1. 全局：健康状态、哪类组件异常、有没有恢复在跑
 ceph health detail          ← 2. 定位：具体是哪块 OSD / 哪个 PG / 哪个 MON / 哪个客户端
 ceph osd find N             ← 3. 找到主机：osd tree / osd metadata / orch ps
 cephadm logs --name <d>     ← 4. 守护进程日志：journald 里的报错
 ceph tell <d> ...           ← 5. 守护进程内部：in-flight 请求、会话、性能计数器
 dmesg / iostat / ping / ss  ← 6. 系统层：盘、网络、内存、时钟

# ceph health detail
HEALTH_WARN clock skew detected on mon.ceph3; 1 osds down; Degraded data redundancy: 2201/66030 objects degraded (3.333%), 45 pgs degraded, 45 pgs undersized; 1 pool(s) nearfull; too many PGs per OSD (312 > max 250)
[WRN] MON_CLOCK_SKEW: clock skew detected on mon.ceph3
    mon.ceph3 clock skew 0.284s > max 0.05s (latency 0.0012s)
[WRN] OSD_DOWN: 1 osds down
    osd.5 (root=default,host=ceph2) is down
[WRN] PG_DEGRADED: Degraded data redundancy: 2201/66030 objects degraded (3.333%), 45 pgs degraded, 45 pgs undersized
    pg 2.1 is stuck undersized for 8m, current state active+undersized+degraded, last acting [3,7]
    ...
[WRN] POOL_NEAR_FULL: 1 pool(s) nearfull
    pool 'rbd01' is nearfull
[WRN] TOO_MANY_PGS: too many PGs per OSD (312 > max 250)

chronyc tracking; systemctl status chrony               # 时钟：在 ceph3 上
ceph osd find 5; cephadm logs --name osd.5 | tail -30  # OSD：是进程还是盘（日志在 ceph2 上看）
dmesg -T | grep -i -E 'error|reset|I/O' | tail
ceph osd pool get-quota rbd01; ceph df detail          # 池容量：是配额还是真满了
ceph osd pool autoscale-status                         # PG 数：谁建了大量 PG、谁关了 autoscaler

ceph osd out 5 && ceph orch osd rm 5 --replace --zap           # 1. 坏盘：确认不是进程问题后按换盘流程处理
systemctl enable --now chrony && chronyc makestep              # 2. 时钟（在 ceph3 上）
ceph osd pool set test-pool-1 pg_autoscale_mode on             # 3. 测试池：删掉，或者重新打开 autoscaler
ceph osd pool set-quota rbd01 max_bytes $((800 * 1024**3))     # 4. 配额：和业务方确认后调整

# ceph health detail
HEALTH_ERR 1 scrub errors; Possible data damage: 1 pg inconsistent
[ERR] OSD_SCRUB_ERRORS: 1 scrub errors
[ERR] PG_DAMAGED: Possible data damage: 1 pg inconsistent
    pg 2.1f is active+clean+inconsistent, acting [1,4,7]

rados list-inconsistent-obj 2.1f --format=json-pretty

{ "epoch": 1245, "inconsistents": [ {
    "object": { "name": "rbd_data.5e3c9a1b2f4d.0000000000000a1f", "snap": "head", "version": 3012 },
    "errors": [], "union_shard_errors": ["read_error"],
    "shards": [
        { "osd": 1, "primary": true,  "errors": [],             "size": 4194304 },
        { "osd": 4, "primary": false, "errors": [],             "size": 4194304 },
        { "osd": 7, "primary": false, "errors": ["read_error"], "size": 4194304 } ] } ] }

ceph osd find 7
dmesg -T | grep -i -E 'medium error|unrecovered read|sector'
cephadm shell -- smartctl -a /dev/sdc | grep -i -E 'reallocated|pending|uncorrectable'

ceph pg repair 2.1f
ceph -w | grep 2.1f          # 看到 repair ok 之后
ceph pg deep-scrub 2.1f      # 再做一次深度 scrub 确认

# ceph health detail
HEALTH_WARN 23 slow ops, oldest one blocked for 67 sec, daemons [osd.4,osd.7] have slow ops.
[WRN] SLOW_OPS: 23 slow ops, oldest one blocked for 67 sec, daemons [osd.4,osd.7] have slow ops.

# ceph osd perf
osd  commit_latency(ms)  apply_latency(ms)
  8                   3                  3
  7                 412                412
  6                   2                  2
  4                   5                  5
...

ceph tell osd.4 dump_historic_slow_ops | jq '.ops[0]'

{ "description": "osd_op(client.84123.0:9921 2.1f ... rbd_data.5e3c9a1b2f4d.00000000000003b2:head [write 16384~4096] ...)",
  "duration": 38.917,
  "type_data": { "flag_point": "commit sent; apply or cleanup", "events": [
      { "event": "initiated",                   "duration": 0 },
      { "event": "queued_for_pg",               "duration": 0.0002 },
      { "event": "started",                     "duration": 0.0004 },
      { "event": "waiting for subops from 1,7", "duration": 0.0009 },
      { "event": "sub_op_commit_rec from 1",    "duration": 0.0021 },
      { "event": "sub_op_commit_rec from 7",    "duration": 38.9031 },
      { "event": "commit_sent",                 "duration": 0.0003 } ] } }

# iostat -x 2 /dev/sdc
Device   r/s    w/s   rkB/s   wkB/s  r_await  w_await  aqu-sz  %util
sdc     12.0   85.5   480.0  6120.0   180.25   962.40   91.33  100.00

ceph osd out 7            # 先止血：让它不再承载 PG
# 数据迁走后，按换盘流程更换

# ceph -s
2026-09-24T03:12:41.118+0000 7f3a1c7fe640  0 monclient(hunting): authenticate timed out after 300
[errno 110] RADOS timed out (error connecting to the cluster)

cephadm ls | jq -r '.[] | select(.name|startswith("mon")) | "\(.name) \(.state)"'
systemctl status ceph-<fsid>@mon.ceph2
cephadm logs --name mon.ceph2 | tail -30
cephadm enter --name mon.ceph1       # 进入还活着的 MON 容器
ceph daemon mon.ceph1 mon_status | jq '{state, quorum, outside_quorum}'

{ "state": "probing", "quorum": [], "outside_quorum": ["ceph1"] }

mon.ceph2@1(peon) e3 reached critical levels of available space on local monitor storage -- shutdown!

# 1. 在 ceph2、ceph3 上腾出空间：清理日志、旧镜像等（不要动 store.db）
journalctl --vacuum-size=500M
podman image prune -a       # 用 Docker 时是 docker image prune -a
# 2. 启动 MON
systemctl start ceph-<fsid>@mon.ceph2
systemctl start ceph-<fsid>@mon.ceph3
ceph -s                     # quorum 恢复
# 3. 集群恢复后压缩 MON 数据库，回收空间
ceph tell mon.ceph2 compact

# ceph health detail
HEALTH_WARN 1 clients failing to respond to capability release; 1 MDSs report slow requests
[WRN] MDS_CLIENT_LATE_RELEASE: 1 clients failing to respond to capability release
    mds.cfs01.ceph2.qwkfzp(mds.0): Client gpu-node-17 failing to respond to capability release client_id: 4305
[WRN] MDS_SLOW_REQUEST: 1 MDSs report slow requests
    mds.cfs01.ceph2.qwkfzp(mds.0): 23 slow requests are blocked > 30 secs

# MDS 在等什么
ceph tell mds.cfs01:0 dump_ops_in_flight | jq '.ops[] | {description, age, flag_point: .type_data.flag_point}' | head -20
# 所有客户端会话：按持有的 caps 数排序
ceph tell mds.cfs01:0 session ls | \
  jq -r '.[] | [.id, .num_caps, .client_metadata.hostname, .inst] | @tsv' | sort -k2 -n -r | head

4305   1843221  gpu-node-17  client.4305 v1:192.168.10.57:0/3172389011
4412   20113    gpu-node-03  client.4412 v1:192.168.10.43:0/2291038475
...

cat /sys/kernel/debug/ceph/*/mdsc | head      # 挂起的 MDS 请求
cat /sys/kernel/debug/ceph/*/osdc | head      # 挂起的 OSD 请求
dmesg -T | grep -i ceph | tail

ceph tell mds.cfs01:0 client evict id=4305
ceph osd blocklist ls                         # 被驱逐的客户端会被加入黑名单

# ceph health detail
HEALTH_WARN 1 large omap objects; 6 slow ops, oldest one blocked for 41 sec, osd.2 has slow ops
[WRN] LARGE_OMAP_OBJECTS: 1 large omap objects
    1 large objects found in pool 'cn-east-1.rgw.buckets.index'
[WRN] SLOW_OPS: 6 slow ops, oldest one blocked for 41 sec, osd.2 has slow ops

# RGW 的请求队列是否堆积（指标名以你的版本实际输出为准）
curl -s http://ceph1:9926/metrics | grep -E 'rgw.*(qlen|qactive)'
# 是不是限流
radosgw-admin ratelimit get --ratelimit-scope=user --uid=team-a
# 索引分片是否过载
radosgw-admin bucket limit check | jq '.[] | .buckets[] | select(.fill_status != "OK")'

{ "bucket": "app-logs", "tenant": "", "num_objects": 48213377, "num_shards": 11,
  "objects_per_shard": 4383034, "fill_status": "OVER 4383%" }

# 选择低峰期；重分片期间该桶的写入可能被短暂阻塞
radosgw-admin bucket reshard --bucket=app-logs --num-shards=499
radosgw-admin reshard status --bucket=app-logs
# 重分片完成后，旧的索引对象会被清理；过几小时再触发一次深度 scrub，确认告警消失
ceph config get client.rgw rgw_dynamic_resharding     # 确认动态重分片已开启