經常使用的ceph調試命令

ceph osd tree
list weight, status and logic structure of all osd 輸入圖片說明node

ceph osd df
list weight, space usage and pg amount of all osd輸入圖片說明app

**ceph osd dump|grep ^osd **
query osd info from osdmap 輸入圖片說明socket

**ceph osd dump|grep ^pool **
query pool info from osdmap 輸入圖片說明spa

ceph pg dump list content of pg map 輸入圖片說明debug

ceph pg map <pg_id> find up and acting osd set of a pgcode

root@node3:~$ ceph pg map 0.ff
osdmap e12 pg 0.ff (0.ff) -> up [0] acting [0]

ceph daemon osd.<index> help
Ceph-osd debug command from admin socket 輸入圖片說明orm

osd_enable_op_tracker=true
debug_optracker=5/5
輸入圖片說明隊列

**ceph daemon osd.0 perf dump **
dump performan statistics圖片

"op_w": 2073643,                           // osd執行的寫操做總數
       "op_w_in_bytes": 8493641728, // osd寫過的數據總長度
       "op_w_rlat": {                     // 4899.396228711秒 / 2073643 = 0.002362699秒 (osd從收到消息到寫操做執行完成且可讀的平均處理時間)
           "avgcount": 2073643,
           "sum": 4899.396228711
       },
       "op_w_latency": {
           "avgcount": ,2073643     // 4903.004405663秒 / 2073643 = 0.002364439秒  (osd從收到消息到寫操做執行完成平均處理時間)
           "sum": 4903.004405663
       },
       "op_w_process_latency": {    // 4793.747818290秒 / 2073643 = 0.002311751秒 (osd從寫操做進入pg隊列後到完成的處理時間)
           "avgcount": 2073643,
           "sum": 4793.747818290
       },
       "op_w_prepare_latency": {    // 535.981493596秒 / 2073643 = 0.000258473秒  (op在pg隊列中的滯留時間)
           "avgcount": 2073643,
           "sum": 535.981493596
       },

ceph daemon osd.0 ops dump inflight ops 輸入圖片說明rem

**ceph pg 0.c5 query **

query pg peering infomation
{
    "state": "remapped+peering",             //PG狀態
    "snap_trimq": "[]",
    "epoch": 21,
    "up": [                                 //up!=acting 因此狀態爲remapped
        0,
        2
    ],
    "acting": [            //acting primary爲osd.0, 在osd.0和osd.1上有pg 0.c5的副本
        0,
        1
    ],
    "info": {             //osd.0上pg 0.c 的信息                   
        "pgid": "0.c5",
        "last_update": "0'0",       //pg log最新記錄編號
        "last_complete": "0'0",     //pg log各副本數據一致
        "log_tail": "0'0",          //pg log最老記錄編號
        "last_user_version": 0,
        "last_backfill": "MAX",
        "last_backfill_bitwise": 0,
        "purged_snaps": "[]",
        "history": {
            "epoch_created": 1,
            "last_epoch_started": 10,
            "last_epoch_clean": 10,       //epoch 10時,pg 0.c5狀態爲clean
            "last_epoch_split": 0,
            "last_epoch_marked_full": 0,
            "same_up_since": 15,
            "same_interval_since": 15,  // epoch 15後,pg 0.c5的up/acting沒有再變化
            "same_primary_since": 9,
            "last_scrub": "0'0",
            "last_scrub_stamp": "2016-09-09 11:51:18.563399",
            "last_deep_scrub": "0'0",
            "last_deep_scrub_stamp": "2016-09-09 11:51:18.563399",
            "last_clean_scrub_stamp": "2016-09-09 11:51:18.563399"
        },
        "stats": {
            "version": "0'0",
            "reported_seq": "14",
            "reported_epoch": "16",
            "state": "remapped+peering",
            "last_fresh": "2016-09-09 11:52:06.425706",
            "last_change": "2016-09-09 11:52:05.351546",
            "last_active": "2016-09-09 11:51:53.364638",
            "last_peered": "2016-09-09 11:51:53.364638",
            "last_clean": "2016-09-09 11:51:18.563399",
            "last_became_active": "2016-09-09 11:51:53.364312",
            "last_became_peered": "2016-09-09 11:51:53.364312",
            "last_unstale": "2016-09-09 11:52:06.425706",
            "last_undegraded": "2016-09-09 11:52:06.425706",
            "last_fullsized": "2016-09-09 11:52:06.425706",
            "mapping_epoch": 15,
            "log_start": "0'0",
            "ondisk_log_start": "0'0",
            "created": 1,
            "last_epoch_clean": 10,
            "parent": "0.0",
            "parent_split_bits": 0,
            "last_scrub": "0'0",
            "last_scrub_stamp": "2016-09-09 11:51:18.563399",
            "last_deep_scrub": "0'0",
            "last_deep_scrub_stamp": "2016-09-09 11:51:18.563399",
            "last_clean_scrub_stamp": "2016-09-09 11:51:18.563399",
            "log_size": 0,
            "ondisk_log_size": 0,
            "stats_invalid": false,
            "dirty_stats_invalid": false,
            "omap_stats_invalid": false,
            "hitset_stats_invalid": false,
            "hitset_bytes_stats_invalid": false,
            "pin_stats_invalid": false,
            "stat_sum": {
                "num_bytes": 0,
                "num_objects": 0,
                "num_object_clones": 0,
                "num_object_copies": 0,
                "num_objects_missing_on_primary": 0,
                "num_objects_missing": 0,
                "num_objects_degraded": 0,
                "num_objects_misplaced": 0,
                "num_objects_unfound": 0,
                "num_objects_dirty": 0,
                "num_whiteouts": 0,
                "num_read": 0,
                "num_read_kb": 0,
                "num_write": 0,
                "num_write_kb": 0,
                "num_scrub_errors": 0,
                "num_shallow_scrub_errors": 0,
                "num_deep_scrub_errors": 0,
                "num_objects_recovered": 0,
                "num_bytes_recovered": 0,
                "num_keys_recovered": 0,
                "num_objects_omap": 0,
                "num_objects_hit_set_archive": 0,
                "num_bytes_hit_set_archive": 0,
                "num_flush": 0,
                "num_flush_kb": 0,
                "num_evict": 0,
                "num_evict_kb": 0,
                "num_promote": 0,
                "num_flush_mode_high": 0,
                "num_flush_mode_low": 0,
                "num_evict_mode_some": 0,
                "num_evict_mode_full": 0,
                "num_objects_pinned": 0
            },
            "up": [
                0,
                2
            ],
            "acting": [
                0,
                1
            ],
            "blocked_by": [
                1
            ],
            "up_primary": 0,
            "acting_primary": 0
        },
        "empty": 1,
        "dne": 0,
        "incomplete": 0,
        "last_epoch_started": 10,
        "hit_set_history": {
            "current_last_update": "0'0",
            "history": []
        }
    },
    "peer_info": [
        {
            "peer": "2",                 //osd.2上pg 0.c 的信息
            "pgid": "0.c5",
            "last_update": "0'0",
            "last_complete": "0'0",
            "log_tail": "0'0",
            "last_user_version": 0,
            "last_backfill": "MAX",
            "last_backfill_bitwise": 0,
            "purged_snaps": "[]",
            "history": {
                "epoch_created": 0,
                "last_epoch_started": 0,
                "last_epoch_clean": 0,
                "last_epoch_split": 0,
                "last_epoch_marked_full": 0,
                "same_up_since": 0,
                "same_interval_since": 0,
                "same_primary_since": 0,
                "last_scrub": "0'0",
                "last_scrub_stamp": "0.000000",
                "last_deep_scrub": "0'0",
                "last_deep_scrub_stamp": "0.000000",
                "last_clean_scrub_stamp": "0.000000"
            },
            "stats": {
                "version": "0'0",
                "reported_seq": "0",
                "reported_epoch": "0",
                "state": "inactive",
                "last_fresh": "0.000000",
                "last_change": "0.000000",
                "last_active": "0.000000",
                "last_peered": "0.000000",
                "last_clean": "0.000000",
                "last_became_active": "0.000000",
                "last_became_peered": "0.000000",
                "last_unstale": "0.000000",
                "last_undegraded": "0.000000",
                "last_fullsized": "0.000000",
                "mapping_epoch": 0,
                "log_start": "0'0",
                "ondisk_log_start": "0'0",
                "created": 0,
                "last_epoch_clean": 0,
                "parent": "0.0",
                "parent_split_bits": 0,
                "last_scrub": "0'0",
                "last_scrub_stamp": "0.000000",
                "last_deep_scrub": "0'0",
                "last_deep_scrub_stamp": "0.000000",
                "last_clean_scrub_stamp": "0.000000",
                "log_size": 0,
                "ondisk_log_size": 0,
                "stats_invalid": false,
                "dirty_stats_invalid": false,
                "omap_stats_invalid": false,
                "hitset_stats_invalid": false,
                "hitset_bytes_stats_invalid": false,
                "pin_stats_invalid": false,
                "stat_sum": {
                    "num_bytes": 0,
                    "num_objects": 0,
                    "num_object_clones": 0,
                    "num_object_copies": 0,
                    "num_objects_missing_on_primary": 0,
                    "num_objects_missing": 0,
                    "num_objects_degraded": 0,
                    "num_objects_misplaced": 0,
                    "num_objects_unfound": 0,
                    "num_objects_dirty": 0,
                    "num_whiteouts": 0,
                    "num_read": 0,
                    "num_read_kb": 0,
                    "num_write": 0,
                    "num_write_kb": 0,
                    "num_scrub_errors": 0,
                    "num_shallow_scrub_errors": 0,
                    "num_deep_scrub_errors": 0,
                    "num_objects_recovered": 0,
                    "num_bytes_recovered": 0,
                    "num_keys_recovered": 0,
                    "num_objects_omap": 0,
                    "num_objects_hit_set_archive": 0,
                    "num_bytes_hit_set_archive": 0,
                    "num_flush": 0,
                    "num_flush_kb": 0,
                    "num_evict": 0,
                    "num_evict_kb": 0,
                    "num_promote": 0,
                    "num_flush_mode_high": 0,
                    "num_flush_mode_low": 0,
                    "num_evict_mode_some": 0,
                    "num_evict_mode_full": 0,
                    "num_objects_pinned": 0
                },
                "up": [],
                "acting": [],
                "blocked_by": [],
                "up_primary": -1,
                "acting_primary": -1
            },
            "empty": 1,
            "dne": 1,
            "incomplete": 0,
            "last_epoch_started": 0,
            "hit_set_history": {
                "current_last_update": "0'0",
                "history": []
            }
        }
    ],
    "recovery_state": [
        {
            "name": "Started\/Primary\/Peering\/GetInfo",
            "enter_time": "2016-09-09 11:52:05.351102",
            "requested_info_from": [
                {
                    "osd": "1"  //peering被osd.1阻塞,沒有向primary osd返回pg info
                }
            ]
        },
        {
            "name": "Started\/Primary\/Peering",
            "enter_time": "2016-09-09 11:52:05.351054",
            "past_intervals": [
                {
                    "first": 9,
                    "last": 11,
                    "maybe_went_rw": 1,
                    "up": [
                        0
                    ],
                    "acting": [
                        0
                    ],
                    "primary": 0,
                    "up_primary": 0
                },
                {
                    "first": 12,
                    "last": 13,
                    "maybe_went_rw": 1,
                    "up": [
                        0,
                        1
                    ],
                    "acting": [
                        0
                    ],
                    "primary": 0,
                    "up_primary": 0
                },
                {
                    "first": 14,
                    "last": 14,
                    "maybe_went_rw": 0,
                    "up": [
                        0,
                        1
                    ],
                    "acting": [
                        0,
                        1
                    ],
                    "primary": 0,
                    "up_primary": 0
                }
            ],
            "probing_osds": [               //peering過程當中須要比較pg info的osd
                "0",
                "1",
                "2"
            ],
            "down_osds_we_would_probe": [],
            "peering_blocked_by": []
        },
        {
            "name": "Started",
            "enter_time": "2016-09-09 11:52:05.350750"
        }
    ],

ceph osd reweight 1 0.5
Before reweight:

root@node3:~$ ceph osd tree
ID WEIGHT  TYPE NAME      UP/DOWN REWEIGHT PRIMARY-AFFINITY 
-1 2.00000 root default                                     
-2 2.00000     host node3                                   
 0 1.00000         osd.0       up  1.00000          1.00000 
 1 1.00000         osd.1       up  1.00000          1.00000
root@node3:~$ ceph osd df
ID WEIGHT  REWEIGHT SIZE  USE   AVAIL %USE  VAR  PGS 
 0 1.00000  1.00000 2098M 1025M 1072M 48.86 1.00 133 
 1 1.00000  1.00000 2098M 1025M 1072M 48.86 1.00 123

After reweight:

root@node3:~$ ceph osd tree
ID WEIGHT  TYPE NAME      UP/DOWN REWEIGHT PRIMARY-AFFINITY 
-1 2.00000 root default                                     
-2 2.00000     host node3                                   
 0 1.00000         osd.0       up  1.00000          1.00000 
 1 1.00000         osd.1       up  0.50000          1.00000
root@node3:~$ ceph osd df
ID WEIGHT  REWEIGHT SIZE  USE   AVAIL %USE  VAR  PGS 
 0 1.00000  1.00000 2098M 1025M 1072M 48.86 1.00 202 
 1 1.00000  0.50000 2098M 1025M 1072M 48.86 1.00  54

ceph daemon osd.1 config show
list all configurable parameters

{
    "name": "osd.1",
    "cluster": "ceph",
    "debug_none": "0\/5",
    "debug_lockdep": "0\/1",
    "debug_context": "0\/1",
    "debug_crush": "1\/1",
    "debug_mds": "1\/5",
    "debug_mds_balancer": "1\/5",
    "debug_mds_locker": "1\/5",
    "debug_mds_log": "1\/5",
    ......

ceph daemon osd.1 config show | grep backfills lookup the value of backfills

root@node3:src(develop)$ ceph daemon osd.1 config show | grep backfills
    "osd_max_backfills": "1",

ceph daemon osd.1 config set osd_max_backfills 2
modify a configurable var value of one osd

root@node3:src(develop)$ ceph daemon osd.1 config show | grep backfills
    "osd_max_backfills": "2",

ceph daemon <path-to-asok> config show
show configs

root@node3:src(develop)$ ceph daemon /var/run/ceph/ceph-osd.0.asok config show | grep backfills
    "osd_max_backfills": "2",

ceph tell osd. injectargs "--osd_max_backfills=2"*
batch modify osds

root@node3:src(develop)$ ceph daemon osd.1 config show | grep backfills
    "osd_max_backfills": "2",
root@node3:src(develop)$ ceph daemon osd.0 config show | grep backfills
    "osd_max_backfills": "2",

ceph pg map 0.4

root@node3:src(develop)$ ./ceph pg map 0.4
osdmap e11 pg 0.4 (0.4) -> up [0,2,1] acting [0,2,1]

ceph osd map <pool> <objectName>

例子以下:
root@node3:src(develop)$ rados -p testpool put testobject /etc/hosts
root@node3:src(develop)$ rados -p testpool ls
testobject
root@node3:src(develop)$ ceph osd map testpool testobject
osdmap e71 pool 'testpool' (6) object 'testobject' -> pg 6.98824931 (6.31) -> up ([5, 4 ,6], p5) acting ([5,4,6], p5)
相關文章
相關標籤/搜索