package ceph import ( "fmt" "os" "path/filepath" "strconv" "strings" "testing" "github.com/stretchr/testify/require" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/testutil" ) const ( epsilon = float64(0.00000001) ) type expectedResult struct { metric string fields map[string]interface{} tags map[string]string } func TestParseSockId(t *testing.T) { s := parseSockID(sockFile(osdPrefix, 1), osdPrefix, sockSuffix) require.Equal(t, "1", s) } func TestParseMonDump(t *testing.T) { c := &Ceph{Log: testutil.Logger{}} dump, err := c.parseDump(monPerfDump) require.NoError(t, err) require.InEpsilon(t, int64(5678670180), dump["cluster"]["osd_kb_used"], epsilon) require.InEpsilon(t, 6866.540527000, dump["paxos"]["store_state_latency.sum"], epsilon) } func TestParseOsdDump(t *testing.T) { c := &Ceph{Log: testutil.Logger{}} dump, err := c.parseDump(osdPerfDump) require.NoError(t, err) require.InEpsilon(t, 552132.109360000, dump["filestore"]["commitcycle_interval.sum"], epsilon) require.InDelta(t, float64(0), dump["mutex-FileJournal::finisher_lock"]["wait.avgcount"], testutil.DefaultDelta) } func TestParseMdsDump(t *testing.T) { c := &Ceph{Log: testutil.Logger{}} dump, err := c.parseDump(mdsPerfDump) require.NoError(t, err) require.InEpsilon(t, 2408386.600934982, dump["mds"]["reply_latency.sum"], epsilon) require.InDelta(t, float64(0), dump["throttle-write_buf_throttle"]["wait.avgcount"], testutil.DefaultDelta) } func TestParseRgwDump(t *testing.T) { c := &Ceph{Log: testutil.Logger{}} dump, err := c.parseDump(rgwPerfDump) require.NoError(t, err) require.InEpsilon(t, 0.002219876, dump["rgw"]["get_initial_lat.sum"], epsilon) require.InDelta(t, float64(0), dump["rgw"]["put_initial_lat.avgcount"], testutil.DefaultDelta) } func TestDecodeStatus(t *testing.T) { acc := &testutil.Accumulator{} err := decodeStatus(acc, clusterStatusDump) require.NoError(t, err) for _, r := range cephStatusResults { acc.AssertContainsTaggedFields(t, r.metric, r.fields, r.tags) } } func TestDecodeStatusNestedOSD(t *testing.T) { acc := &testutil.Accumulator{} err := decodeStatus(acc, clusterStatusDumpNestedOSD) require.NoError(t, err) for _, r := range cephStatusResultsNestedOSD { acc.AssertContainsTaggedFields(t, r.metric, r.fields, r.tags) } } func TestDecodeDf(t *testing.T) { acc := &testutil.Accumulator{} err := decodeDf(acc, cephDFDump) require.NoError(t, err) for _, r := range cephDfResults { acc.AssertContainsTaggedFields(t, r.metric, r.fields, r.tags) } } func TestDecodeOSDPoolStats(t *testing.T) { acc := &testutil.Accumulator{} err := decodeOsdPoolStats(acc, cephODSPoolStatsDump) require.NoError(t, err) for _, r := range cephOSDPoolStatsResults { acc.AssertContainsTaggedFields(t, r.metric, r.fields, r.tags) } } func TestGather(t *testing.T) { saveFind := findSockets saveDump := perfDump defer func() { findSockets = saveFind perfDump = saveDump }() findSockets = func(*Ceph) ([]*socket, error) { return []*socket{{"osd.1", typeOsd, ""}}, nil } perfDump = func(string, *socket) (string, error) { return osdPerfDump, nil } acc := &testutil.Accumulator{} c := &Ceph{} require.NoError(t, c.Gather(acc)) } func TestParseSchema(t *testing.T) { schemaMap, err := parseSchema(osdRawSchema) require.NoError(t, err) // Test Gauge require.Equal(t, telegraf.Counter, schemaMap["osd"]["op"], "op should be a Counter") // Test Counter require.Equal(t, telegraf.Gauge, schemaMap["osd"]["op_wip"], "op_wip should be a Gauge") // Test LongRunAvg require.Equal(t, telegraf.Counter, schemaMap["osd"]["op_latency.avgcount"], "op_latency.avgcount should be a Counter") require.Equal(t, telegraf.Counter, schemaMap["osd"]["op_latency.sum"], "op_latency.sum should be a Counter") require.Equal(t, telegraf.Gauge, schemaMap["osd"]["op_latency.avgtime"], "op_latency.avgtime should be a Gauge") } func TestFindSockets(t *testing.T) { tmpdir := t.TempDir() c := &Ceph{ CephBinary: "foo", OsdPrefix: "ceph-osd", MonPrefix: "ceph-mon", MdsPrefix: "ceph-mds", RgwPrefix: "ceph-client", SocketDir: tmpdir, SocketSuffix: "asok", CephUser: "client.admin", CephConfig: "/etc/ceph/ceph.conf", GatherAdminSocketStats: true, GatherClusterStats: false, } for _, st := range sockTestParams { require.NoError(t, createTestFiles(tmpdir, st)) sockets, err := findSockets(c) require.NoError(t, err) for i := 1; i <= st.osds; i++ { assertFoundSocket(t, tmpdir, typeOsd, i, sockets) } for i := 1; i <= st.mons; i++ { assertFoundSocket(t, tmpdir, typeMon, i, sockets) } for i := 1; i <= st.mdss; i++ { assertFoundSocket(t, tmpdir, typeMds, i, sockets) } for i := 1; i <= st.rgws; i++ { assertFoundSocket(t, tmpdir, typeRgw, i, sockets) } require.NoError(t, cleanupTestFiles(tmpdir, st)) } } func assertFoundSocket(t *testing.T, dir, sockType string, i int, sockets []*socket) { var prefix string if sockType == typeOsd { prefix = osdPrefix } else if sockType == typeMds { prefix = mdsPrefix } else if sockType == typeRgw { prefix = rgwPrefix } else { prefix = monPrefix } expected := filepath.Join(dir, sockFile(prefix, i)) found := false for _, s := range sockets { fmt.Printf("Checking %s\n", s.socket) if s.socket == expected { found = true require.Equal(t, s.sockType, sockType, "Unexpected socket type for %q", s) require.Equal(t, s.sockID, strconv.Itoa(i)) } } require.True(t, found, "Did not find socket: %s", expected) } func sockFile(prefix string, i int) string { return strings.Join([]string{prefix, strconv.Itoa(i), sockSuffix}, ".") } func createTestFiles(dir string, st *SockTest) error { writeFile := func(prefix string, i int) error { f := sockFile(prefix, i) fpath := filepath.Join(dir, f) return os.WriteFile(fpath, []byte(""), 0640) } return tstFileApply(st, writeFile) } func cleanupTestFiles(dir string, st *SockTest) error { rmFile := func(prefix string, i int) error { f := sockFile(prefix, i) fpath := filepath.Join(dir, f) return os.Remove(fpath) } return tstFileApply(st, rmFile) } func tstFileApply(st *SockTest, fn func(string, int) error) error { for i := 1; i <= st.osds; i++ { if err := fn(osdPrefix, i); err != nil { return err } } for i := 1; i <= st.mons; i++ { if err := fn(monPrefix, i); err != nil { return err } } for i := 1; i <= st.mdss; i++ { if err := fn(mdsPrefix, i); err != nil { return err } } for i := 1; i <= st.rgws; i++ { if err := fn(rgwPrefix, i); err != nil { return err } } return nil } type SockTest struct { osds int mons int mdss int rgws int } var sockTestParams = []*SockTest{ { osds: 2, mons: 2, mdss: 2, rgws: 2, }, { mons: 1, }, { osds: 1, }, { mdss: 1, }, { rgws: 1, }, {}, } var monPerfDump = ` { "cluster": { "num_mon": 2, "num_mon_quorum": 2, "num_osd": 26, "num_osd_up": 26, "num_osd_in": 26, "osd_epoch": 3306, "osd_kb": 11487846448, "osd_kb_used": 5678670180, "osd_kb_avail": 5809176268, "num_pool": 12, "num_pg": 768, "num_pg_active_clean": 768, "num_pg_active": 768, "num_pg_peering": 0, "num_object": 397616, "num_object_degraded": 0, "num_object_unfound": 0, "num_bytes": 2917848227467, "num_mds_up": 0, "num_mds_in": 0, "num_mds_failed": 0, "mds_epoch": 1}, "leveldb": { "leveldb_get": 321950312, "leveldb_transaction": 18729922, "leveldb_compact": 0, "leveldb_compact_range": 74141, "leveldb_compact_queue_merge": 0, "leveldb_compact_queue_len": 0}, "mon": {}, "paxos": { "start_leader": 0, "start_peon": 1, "restart": 4, "refresh": 9363435, "refresh_latency": { "avgcount": 9363435, "sum": 5378.794002000}, "begin": 9363435, "begin_keys": { "avgcount": 0, "sum": 0}, "begin_bytes": { "avgcount": 9363435, "sum": 110468605489}, "begin_latency": { "avgcount": 9363435, "sum": 5850.060682000}, "commit": 9363435, "commit_keys": { "avgcount": 0, "sum": 0}, "commit_bytes": { "avgcount": 0, "sum": 0}, "commit_latency": { "avgcount": 0, "sum": 0.000000000}, "collect": 1, "collect_keys": { "avgcount": 1, "sum": 1}, "collect_bytes": { "avgcount": 1, "sum": 24}, "collect_latency": { "avgcount": 1, "sum": 0.000280000}, "collect_uncommitted": 0, "collect_timeout": 0, "accept_timeout": 0, "lease_ack_timeout": 0, "lease_timeout": 0, "store_state": 9363435, "store_state_keys": { "avgcount": 9363435, "sum": 176572789}, "store_state_bytes": { "avgcount": 9363435, "sum": 216355887217}, "store_state_latency": { "avgcount": 9363435, "sum": 6866.540527000}, "share_state": 0, "share_state_keys": { "avgcount": 0, "sum": 0}, "share_state_bytes": { "avgcount": 0, "sum": 0}, "new_pn": 0, "new_pn_latency": { "avgcount": 0, "sum": 0.000000000}}, "throttle-mon_client_bytes": { "val": 246, "max": 104857600, "get": 896030, "get_sum": 45854374, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 896026, "put_sum": 45854128, "wait": { "avgcount": 0, "sum": 0.000000000}}, "throttle-mon_daemon_bytes": { "val": 0, "max": 419430400, "get": 2773768, "get_sum": 3627676976, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 2773768, "put_sum": 3627676976, "wait": { "avgcount": 0, "sum": 0.000000000}}, "throttle-msgr_dispatch_throttler-mon": { "val": 0, "max": 104857600, "get": 34504949, "get_sum": 226860281124, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 34504949, "put_sum": 226860281124, "wait": { "avgcount": 0, "sum": 0.000000000}}} ` var osdPerfDump = ` { "WBThrottle": { "bytes_dirtied": 28405539, "bytes_wb": 0, "ios_dirtied": 93, "ios_wb": 0, "inodes_dirtied": 86, "inodes_wb": 0}, "filestore": { "journal_queue_max_ops": 0, "journal_queue_ops": 0, "journal_ops": 1108008, "journal_queue_max_bytes": 0, "journal_queue_bytes": 0, "journal_bytes": 73233416196, "journal_latency": { "avgcount": 1108008, "sum": 290.981036000}, "journal_wr": 1091866, "journal_wr_bytes": { "avgcount": 1091866, "sum": 74925682688}, "journal_full": 0, "committing": 0, "commitcycle": 110389, "commitcycle_interval": { "avgcount": 110389, "sum": 552132.109360000}, "commitcycle_latency": { "avgcount": 110389, "sum": 178.657804000}, "op_queue_max_ops": 50, "op_queue_ops": 0, "ops": 1108008, "op_queue_max_bytes": 104857600, "op_queue_bytes": 0, "bytes": 73226768148, "apply_latency": { "avgcount": 1108008, "sum": 947.742722000}, "queue_transaction_latency_avg": { "avgcount": 1108008, "sum": 0.511327000}}, "leveldb": { "leveldb_get": 4361221, "leveldb_transaction": 4351276, "leveldb_compact": 0, "leveldb_compact_range": 0, "leveldb_compact_queue_merge": 0, "leveldb_compact_queue_len": 0}, "mutex-FileJournal::completions_lock": { "wait": { "avgcount": 0, "sum": 0.000000000}}, "mutex-FileJournal::finisher_lock": { "wait": { "avgcount": 0, "sum": 0.000000000}}, "mutex-FileJournal::write_lock": { "wait": { "avgcount": 0, "sum": 0.000000000}}, "mutex-FileJournal::writeq_lock": { "wait": { "avgcount": 0, "sum": 0.000000000}}, "mutex-JOS::ApplyManager::apply_lock": { "wait": { "avgcount": 0, "sum": 0.000000000}}, "mutex-JOS::ApplyManager::com_lock": { "wait": { "avgcount": 0, "sum": 0.000000000}}, "mutex-JOS::SubmitManager::lock": { "wait": { "avgcount": 0, "sum": 0.000000000}}, "mutex-WBThrottle::lock": { "wait": { "avgcount": 0, "sum": 0.000000000}}, "objecter": { "op_active": 0, "op_laggy": 0, "op_send": 0, "op_send_bytes": 0, "op_resend": 0, "op_ack": 0, "op_commit": 0, "op": 0, "op_r": 0, "op_w": 0, "op_rmw": 0, "op_pg": 0, "osdop_stat": 0, "osdop_create": 0, "osdop_read": 0, "osdop_write": 0, "osdop_writefull": 0, "osdop_append": 0, "osdop_zero": 0, "osdop_truncate": 0, "osdop_delete": 0, "osdop_mapext": 0, "osdop_sparse_read": 0, "osdop_clonerange": 0, "osdop_getxattr": 0, "osdop_setxattr": 0, "osdop_cmpxattr": 0, "osdop_rmxattr": 0, "osdop_resetxattrs": 0, "osdop_tmap_up": 0, "osdop_tmap_put": 0, "osdop_tmap_get": 0, "osdop_call": 0, "osdop_watch": 0, "osdop_notify": 0, "osdop_src_cmpxattr": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_other": 0, "linger_active": 0, "linger_send": 0, "linger_resend": 0, "poolop_active": 0, "poolop_send": 0, "poolop_resend": 0, "poolstat_active": 0, "poolstat_send": 0, "poolstat_resend": 0, "statfs_active": 0, "statfs_send": 0, "statfs_resend": 0, "command_active": 0, "command_send": 0, "command_resend": 0, "map_epoch": 3300, "map_full": 0, "map_inc": 3293, "osd_sessions": 0, "osd_session_open": 0, "osd_session_close": 0, "osd_laggy": 0}, "osd": { "opq": 0, "op_wip": 0, "op": 23939, "op_in_bytes": 1245903961, "op_out_bytes": 29103083856, "op_latency": { "avgcount": 23939, "sum": 440.192015000}, "op_process_latency": { "avgcount": 23939, "sum": 30.170685000}, "op_r": 23112, "op_r_out_bytes": 29103056146, "op_r_latency": { "avgcount": 23112, "sum": 19.373526000}, "op_r_process_latency": { "avgcount": 23112, "sum": 14.625928000}, "op_w": 549, "op_w_in_bytes": 1245804358, "op_w_rlat": { "avgcount": 549, "sum": 17.022299000}, "op_w_latency": { "avgcount": 549, "sum": 418.494610000}, "op_w_process_latency": { "avgcount": 549, "sum": 13.316555000}, "op_rw": 278, "op_rw_in_bytes": 99603, "op_rw_out_bytes": 27710, "op_rw_rlat": { "avgcount": 278, "sum": 2.213785000}, "op_rw_latency": { "avgcount": 278, "sum": 2.323879000}, "op_rw_process_latency": { "avgcount": 278, "sum": 2.228202000}, "subop": 1074774, "subop_in_bytes": 26841811636, "subop_latency": { "avgcount": 1074774, "sum": 745.509160000}, "subop_w": 0, "subop_w_in_bytes": 26841811636, "subop_w_latency": { "avgcount": 1074774, "sum": 745.509160000}, "subop_pull": 0, "subop_pull_latency": { "avgcount": 0, "sum": 0.000000000}, "subop_push": 0, "subop_push_in_bytes": 0, "subop_push_latency": { "avgcount": 0, "sum": 0.000000000}, "pull": 0, "push": 28, "push_out_bytes": 103483392, "push_in": 0, "push_in_bytes": 0, "recovery_ops": 15, "loadavg": 202, "buffer_bytes": 0, "numpg": 18, "numpg_primary": 8, "numpg_replica": 10, "numpg_stray": 0, "heartbeat_to_peers": 10, "heartbeat_from_peers": 0, "map_messages": 7413, "map_message_epochs": 9792, "map_message_epoch_dups": 10105, "messages_delayed_for_map": 83, "stat_bytes": 102123175936, "stat_bytes_used": 49961820160, "stat_bytes_avail": 52161355776, "copyfrom": 0, "tier_promote": 0, "tier_flush": 0, "tier_flush_fail": 0, "tier_try_flush": 0, "tier_try_flush_fail": 0, "tier_evict": 0, "tier_whiteout": 0, "tier_dirty": 230, "tier_clean": 0, "tier_delay": 0, "agent_wake": 0, "agent_skip": 0, "agent_flush": 0, "agent_evict": 0}, "recoverystate_perf": { "initial_latency": { "avgcount": 473, "sum": 0.027207000}, "started_latency": { "avgcount": 1480, "sum": 9854902.397648000}, "reset_latency": { "avgcount": 1953, "sum": 0.096206000}, "start_latency": { "avgcount": 1953, "sum": 0.059947000}, "primary_latency": { "avgcount": 765, "sum": 4688922.186935000}, "peering_latency": { "avgcount": 704, "sum": 1668.652135000}, "backfilling_latency": { "avgcount": 0, "sum": 0.000000000}, "waitremotebackfillreserved_latency": { "avgcount": 0, "sum": 0.000000000}, "waitlocalbackfillreserved_latency": { "avgcount": 0, "sum": 0.000000000}, "notbackfilling_latency": { "avgcount": 0, "sum": 0.000000000}, "repnotrecovering_latency": { "avgcount": 462, "sum": 5158922.114600000}, "repwaitrecoveryreserved_latency": { "avgcount": 15, "sum": 0.008275000}, "repwaitbackfillreserved_latency": { "avgcount": 1, "sum": 0.000095000}, "RepRecovering_latency": { "avgcount": 16, "sum": 2274.944727000}, "activating_latency": { "avgcount": 514, "sum": 261.008520000}, "waitlocalrecoveryreserved_latency": { "avgcount": 20, "sum": 0.175422000}, "waitremoterecoveryreserved_latency": { "avgcount": 20, "sum": 0.682778000}, "recovering_latency": { "avgcount": 20, "sum": 0.697551000}, "recovered_latency": { "avgcount": 511, "sum": 0.011038000}, "clean_latency": { "avgcount": 503, "sum": 4686961.154278000}, "active_latency": { "avgcount": 506, "sum": 4687223.640464000}, "replicaactive_latency": { "avgcount": 446, "sum": 5161197.078966000}, "stray_latency": { "avgcount": 794, "sum": 4805.105128000}, "getinfo_latency": { "avgcount": 704, "sum": 1138.477937000}, "getlog_latency": { "avgcount": 678, "sum": 0.036393000}, "waitactingchange_latency": { "avgcount": 69, "sum": 59.172893000}, "incomplete_latency": { "avgcount": 0, "sum": 0.000000000}, "getmissing_latency": { "avgcount": 609, "sum": 0.012288000}, "waitupthru_latency": { "avgcount": 576, "sum": 530.106999000}}, "throttle-filestore_bytes": { "val": 0, "max": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000}}, "throttle-filestore_ops": { "val": 0, "max": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000}}, "throttle-msgr_dispatch_throttler-client": { "val": 0, "max": 104857600, "get": 130730, "get_sum": 1246039872, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 130730, "put_sum": 1246039872, "wait": { "avgcount": 0, "sum": 0.000000000}}, "throttle-msgr_dispatch_throttler-cluster": { "val": 0, "max": 104857600, "get": 1108033, "get_sum": 71277949992, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 1108033, "put_sum": 71277949992, "wait": { "avgcount": 0, "sum": 0.000000000}}, "throttle-msgr_dispatch_throttler-hb_back_server": { "val": 0, "max": 104857600, "get": 18320575, "get_sum": 861067025, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 18320575, "put_sum": 861067025, "wait": { "avgcount": 0, "sum": 0.000000000}}, "throttle-msgr_dispatch_throttler-hb_front_server": { "val": 0, "max": 104857600, "get": 18320575, "get_sum": 861067025, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 18320575, "put_sum": 861067025, "wait": { "avgcount": 0, "sum": 0.000000000}}, "throttle-msgr_dispatch_throttler-hbclient": { "val": 0, "max": 104857600, "get": 40479394, "get_sum": 1902531518, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 40479394, "put_sum": 1902531518, "wait": { "avgcount": 0, "sum": 0.000000000}}, "throttle-msgr_dispatch_throttler-ms_objecter": { "val": 0, "max": 104857600, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000}}, "throttle-objecter_bytes": { "val": 0, "max": 104857600, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000}}, "throttle-objecter_ops": { "val": 0, "max": 1024, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000}}, "throttle-osd_client_bytes": { "val": 0, "max": 524288000, "get": 24241, "get_sum": 1241992581, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 25958, "put_sum": 1241992581, "wait": { "avgcount": 0, "sum": 0.000000000}}, "throttle-osd_client_messages": { "val": 0, "max": 100, "get": 49214, "get_sum": 49214, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 49214, "put_sum": 49214, "wait": { "avgcount": 0, "sum": 0.000000000}}} ` var mdsPerfDump = ` { "AsyncMessenger::Worker-0": { "msgr_recv_messages": 2723536628, "msgr_send_messages": 1160771414, "msgr_recv_bytes": 1112936719134, "msgr_send_bytes": 1368194904867, "msgr_created_connections": 18281, "msgr_active_connections": 83, "msgr_running_total_time": 109001.938705141, "msgr_running_send_time": 33686.215323581, "msgr_running_recv_time": 8374950.111041426, "msgr_running_fast_dispatch_time": 5828.083761243 }, "AsyncMessenger::Worker-1": { "msgr_recv_messages": 1426105165, "msgr_send_messages": 783174767, "msgr_recv_bytes": 800620150187, "msgr_send_bytes": 1394738277392, "msgr_created_connections": 17677, "msgr_active_connections": 100, "msgr_running_total_time": 70660.929329800, "msgr_running_send_time": 24190.940207198, "msgr_running_recv_time": 3920894.209204916, "msgr_running_fast_dispatch_time": 8206.816536602 }, "AsyncMessenger::Worker-2": { "msgr_recv_messages": 3471200310, "msgr_send_messages": 2757725529, "msgr_recv_bytes": 1331676471794, "msgr_send_bytes": 2593968875674, "msgr_created_connections": 16714, "msgr_active_connections": 73, "msgr_running_total_time": 167020.893916556, "msgr_running_send_time": 61197.682840176, "msgr_running_recv_time": 5816036.495319415, "msgr_running_fast_dispatch_time": 8581.768789481 }, "finisher-PurgeQueue": { "queue_len": 0, "complete_latency": { "avgcount": 20170260, "sum": 70213.859039869, "avgtime": 0.003481058 } }, "mds": { "request": 2167457412, "reply": 2167457403, "reply_latency": { "avgcount": 2167457403, "sum": 2408386.600934982, "avgtime": 0.001111157 }, "forward": 0, "dir_fetch": 585012985, "dir_commit": 58926158, "dir_split": 8, "dir_merge": 7, "inode_max": 2147483647, "inodes": 39604287, "inodes_top": 9743493, "inodes_bottom": 29063656, "inodes_pin_tail": 797138, "inodes_pinned": 25685011, "inodes_expired": 1302542128, "inodes_with_caps": 4517329, "caps": 6370838, "subtrees": 2, "traverse": 2426357623, "traverse_hit": 2202314009, "traverse_forward": 0, "traverse_discover": 0, "traverse_dir_fetch": 35332112, "traverse_remote_ino": 0, "traverse_lock": 4371557, "load_cent": 1966748, "q": 976, "exported": 0, "exported_inodes": 0, "imported": 0, "imported_inodes": 0, "openino_dir_fetch": 22725418, "openino_backtrace_fetch": 6, "openino_peer_discover": 0 }, "mds_cache": { "num_strays": 384, "num_strays_delayed": 0, "num_strays_enqueuing": 0, "strays_created": 29140050, "strays_enqueued": 29134399, "strays_reintegrated": 10171, "strays_migrated": 0, "num_recovering_processing": 0, "num_recovering_enqueued": 0, "num_recovering_prioritized": 0, "recovery_started": 229, "recovery_completed": 229, "ireq_enqueue_scrub": 0, "ireq_exportdir": 0, "ireq_flush": 0, "ireq_fragmentdir": 15, "ireq_fragstats": 0, "ireq_inodestats": 0 }, "mds_log": { "evadd": 1920368707, "evex": 1920372003, "evtrm": 1920372003, "ev": 106627, "evexg": 0, "evexd": 4369, "segadd": 2247990, "segex": 2247995, "segtrm": 2247995, "seg": 123, "segexg": 0, "segexd": 5, "expos": 24852063335817, "wrpos": 24852205446582, "rdpos": 22044255640175, "jlat": { "avgcount": 182241259, "sum": 1732094.198366820, "avgtime": 0.009504402 }, "replayed": 109923 }, "mds_mem": { "ino": 39604292, "ino+": 1307214891, "ino-": 1267610599, "dir": 22827008, "dir+": 591593031, "dir-": 568766023, "dn": 39604761, "dn+": 1376976677, "dn-": 1337371916, "cap": 6370838, "cap+": 1720930015, "cap-": 1714559177, "rss": 167723320, "heap": 322260, "buf": 0 }, "mds_server": { "dispatch_client_request": 2932764331, "dispatch_server_request": 0, "handle_client_request": 2167457412, "handle_client_session": 10929454, "handle_slave_request": 0, "req_create_latency": { "avgcount": 30590326, "sum": 23887.274170412, "avgtime": 0.000780876 }, "req_getattr_latency": { "avgcount": 124767480, "sum": 718160.497644305, "avgtime": 0.005755991 }, "req_getfilelock_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "req_link_latency": { "avgcount": 5636, "sum": 2.371499732, "avgtime": 0.000420777 }, "req_lookup_latency": { "avgcount": 474590034, "sum": 452548.849373476, "avgtime": 0.000953557 }, "req_lookuphash_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "req_lookupino_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "req_lookupname_latency": { "avgcount": 9794, "sum": 54.118496591, "avgtime": 0.005525678 }, "req_lookupparent_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "req_lookupsnap_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "req_lssnap_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "req_mkdir_latency": { "avgcount": 13394317, "sum": 13025.982105531, "avgtime": 0.000972500 }, "req_mknod_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "req_mksnap_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "req_open_latency": { "avgcount": 32849768, "sum": 12862.382994977, "avgtime": 0.000391551 }, "req_readdir_latency": { "avgcount": 654394394, "sum": 715669.609601541, "avgtime": 0.001093636 }, "req_rename_latency": { "avgcount": 6058807, "sum": 2126.232719555, "avgtime": 0.000350932 }, "req_renamesnap_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "req_rmdir_latency": { "avgcount": 1901530, "sum": 4064.121157858, "avgtime": 0.002137290 }, "req_rmsnap_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "req_rmxattr_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "req_setattr_latency": { "avgcount": 37051209, "sum": 171198.037329531, "avgtime": 0.004620578 }, "req_setdirlayout_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "req_setfilelock_latency": { "avgcount": 765439143, "sum": 262660.582883819, "avgtime": 0.000343150 }, "req_setlayout_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "req_setxattr_latency": { "avgcount": 41572, "sum": 7.273371375, "avgtime": 0.000174958 }, "req_symlink_latency": { "avgcount": 329, "sum": 0.117859965, "avgtime": 0.000358236 }, "req_unlink_latency": { "avgcount": 26363064, "sum": 32119.149726314, "avgtime": 0.001218339 }, "cap_revoke_eviction": 0 }, "mds_sessions": { "session_count": 80, "session_add": 90, "session_remove": 10, "sessions_open": 80, "sessions_stale": 0, "total_load": 112490, "average_load": 1406, "avg_session_uptime": 2221807 }, "objecter": { "op_active": 0, "op_laggy": 0, "op_send": 955060080, "op_send_bytes": 3178832110019, "op_resend": 67, "op_reply": 955060013, "op": 955060013, "op_r": 585982837, "op_w": 369077176, "op_rmw": 0, "op_pg": 0, "osdop_stat": 45924375, "osdop_create": 31162274, "osdop_read": 969513, "osdop_write": 183211164, "osdop_writefull": 1063233, "osdop_writesame": 0, "osdop_append": 0, "osdop_zero": 2, "osdop_truncate": 8, "osdop_delete": 60594735, "osdop_mapext": 0, "osdop_sparse_read": 0, "osdop_clonerange": 0, "osdop_getxattr": 584941886, "osdop_setxattr": 62324548, "osdop_cmpxattr": 0, "osdop_rmxattr": 0, "osdop_resetxattrs": 0, "osdop_tmap_up": 0, "osdop_tmap_put": 0, "osdop_tmap_get": 0, "osdop_call": 0, "osdop_watch": 0, "osdop_notify": 0, "osdop_src_cmpxattr": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_other": 32053182, "linger_active": 0, "linger_send": 0, "linger_resend": 0, "linger_ping": 0, "poolop_active": 0, "poolop_send": 0, "poolop_resend": 0, "poolstat_active": 0, "poolstat_send": 0, "poolstat_resend": 0, "statfs_active": 0, "statfs_send": 0, "statfs_resend": 0, "command_active": 0, "command_send": 0, "command_resend": 0, "map_epoch": 66793, "map_full": 0, "map_inc": 1762, "osd_sessions": 120, "osd_session_open": 52554, "osd_session_close": 52434, "osd_laggy": 0, "omap_wr": 106692727, "omap_rd": 1170026044, "omap_del": 5674762 }, "purge_queue": { "pq_executing_ops": 0, "pq_executing": 0, "pq_executed": 29134399 }, "throttle-msgr_dispatch_throttler-mds": { "val": 0, "max": 104857600, "get_started": 0, "get": 7620842095, "get_sum": 2681291022887, "get_or_fail_fail": 53, "get_or_fail_success": 7620842095, "take": 0, "take_sum": 0, "put": 7620842095, "put_sum": 2681291022887, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_bytes": { "val": 0, "max": 104857600, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 955060013, "take_sum": 3172776432475, "put": 862340641, "put_sum": 3172776432475, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_ops": { "val": 0, "max": 1024, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 955060013, "take_sum": 955060013, "put": 955060013, "put_sum": 955060013, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-write_buf_throttle": { "val": 0, "max": 3758096384, "get_started": 0, "get": 29134399, "get_sum": 3160498139, "get_or_fail_fail": 0, "get_or_fail_success": 29134399, "take": 0, "take_sum": 0, "put": 969905, "put_sum": 3160498139, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-write_buf_throttle-0x561894f0b8e0": { "val": 286270, "max": 3758096384, "get_started": 0, "get": 1920368707, "get_sum": 2807949805409, "get_or_fail_fail": 0, "get_or_fail_success": 1920368707, "take": 0, "take_sum": 0, "put": 182241259, "put_sum": 2807949519139, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } } } ` var rgwPerfDump = ` { "AsyncMessenger::Worker-0": { "msgr_recv_messages": 10684185, "msgr_send_messages": 13448962, "msgr_recv_bytes": 2622531258, "msgr_send_bytes": 4195038384, "msgr_created_connections": 8029, "msgr_active_connections": 3, "msgr_running_total_time": 3249.441108544, "msgr_running_send_time": 739.821446096, "msgr_running_recv_time": 310.354319110, "msgr_running_fast_dispatch_time": 1915.410317430 }, "AsyncMessenger::Worker-1": { "msgr_recv_messages": 2137773, "msgr_send_messages": 3850070, "msgr_recv_bytes": 503824366, "msgr_send_bytes": 1130107261, "msgr_created_connections": 11030, "msgr_active_connections": 1, "msgr_running_total_time": 445.055291782, "msgr_running_send_time": 227.817750758, "msgr_running_recv_time": 78.974093226, "msgr_running_fast_dispatch_time": 47.587740615 }, "AsyncMessenger::Worker-2": { "msgr_recv_messages": 2809014, "msgr_send_messages": 4126613, "msgr_recv_bytes": 653093470, "msgr_send_bytes": 1022041970, "msgr_created_connections": 14810, "msgr_active_connections": 5, "msgr_running_total_time": 453.384703728, "msgr_running_send_time": 208.580910390, "msgr_running_recv_time": 80.075306670, "msgr_running_fast_dispatch_time": 46.854112208 }, "cct": { "total_workers": 0, "unhealthy_workers": 0 }, "finisher-radosclient": { "queue_len": 0, "complete_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "finisher-radosclient-0x55994098e460": { "queue_len": 0, "complete_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "finisher-radosclient-0x5599409901c0": { "queue_len": 0, "complete_latency": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "mempool": { "bloom_filter_bytes": 0, "bloom_filter_items": 0, "bluestore_alloc_bytes": 0, "bluestore_alloc_items": 0, "bluestore_cache_data_bytes": 0, "bluestore_cache_data_items": 0, "bluestore_cache_onode_bytes": 0, "bluestore_cache_onode_items": 0, "bluestore_cache_other_bytes": 0, "bluestore_cache_other_items": 0, "bluestore_fsck_bytes": 0, "bluestore_fsck_items": 0, "bluestore_txc_bytes": 0, "bluestore_txc_items": 0, "bluestore_writing_deferred_bytes": 0, "bluestore_writing_deferred_items": 0, "bluestore_writing_bytes": 0, "bluestore_writing_items": 0, "bluefs_bytes": 0, "bluefs_items": 0, "buffer_anon_bytes": 258469, "buffer_anon_items": 201, "buffer_meta_bytes": 0, "buffer_meta_items": 0, "osd_bytes": 0, "osd_items": 0, "osd_mapbl_bytes": 0, "osd_mapbl_items": 0, "osd_pglog_bytes": 0, "osd_pglog_items": 0, "osdmap_bytes": 74448, "osdmap_items": 732, "osdmap_mapping_bytes": 0, "osdmap_mapping_items": 0, "pgmap_bytes": 0, "pgmap_items": 0, "mds_co_bytes": 0, "mds_co_items": 0, "unittest_1_bytes": 0, "unittest_1_items": 0, "unittest_2_bytes": 0, "unittest_2_items": 0 }, "objecter": { "op_active": 0, "op_laggy": 0, "op_send": 9377910, "op_send_bytes": 312, "op_resend": 0, "op_reply": 9377904, "op": 9377910, "op_r": 2755291, "op_w": 6622619, "op_rmw": 0, "op_pg": 0, "osdop_stat": 2755258, "osdop_create": 8, "osdop_read": 25, "osdop_write": 0, "osdop_writefull": 0, "osdop_writesame": 0, "osdop_append": 0, "osdop_zero": 0, "osdop_truncate": 0, "osdop_delete": 0, "osdop_mapext": 0, "osdop_sparse_read": 0, "osdop_clonerange": 0, "osdop_getxattr": 0, "osdop_setxattr": 0, "osdop_cmpxattr": 0, "osdop_rmxattr": 0, "osdop_resetxattrs": 0, "osdop_call": 0, "osdop_watch": 6622611, "osdop_notify": 0, "osdop_src_cmpxattr": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_other": 2755266, "linger_active": 8, "linger_send": 35, "linger_resend": 27, "linger_ping": 6622576, "poolop_active": 0, "poolop_send": 0, "poolop_resend": 0, "poolstat_active": 0, "poolstat_send": 0, "poolstat_resend": 0, "statfs_active": 0, "statfs_send": 0, "statfs_resend": 0, "command_active": 0, "command_send": 0, "command_resend": 0, "map_epoch": 1064, "map_full": 0, "map_inc": 106, "osd_sessions": 8, "osd_session_open": 11928, "osd_session_close": 11920, "osd_laggy": 5, "omap_wr": 0, "omap_rd": 0, "omap_del": 0 }, "objecter-0x55994098e500": { "op_active": 0, "op_laggy": 0, "op_send": 827839, "op_send_bytes": 0, "op_resend": 0, "op_reply": 827839, "op": 827839, "op_r": 0, "op_w": 827839, "op_rmw": 0, "op_pg": 0, "osdop_stat": 0, "osdop_create": 0, "osdop_read": 0, "osdop_write": 0, "osdop_writefull": 0, "osdop_writesame": 0, "osdop_append": 0, "osdop_zero": 0, "osdop_truncate": 0, "osdop_delete": 0, "osdop_mapext": 0, "osdop_sparse_read": 0, "osdop_clonerange": 0, "osdop_getxattr": 0, "osdop_setxattr": 0, "osdop_cmpxattr": 0, "osdop_rmxattr": 0, "osdop_resetxattrs": 0, "osdop_call": 0, "osdop_watch": 827839, "osdop_notify": 0, "osdop_src_cmpxattr": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_other": 0, "linger_active": 1, "linger_send": 3, "linger_resend": 2, "linger_ping": 827836, "poolop_active": 0, "poolop_send": 0, "poolop_resend": 0, "poolstat_active": 0, "poolstat_send": 0, "poolstat_resend": 0, "statfs_active": 0, "statfs_send": 0, "statfs_resend": 0, "command_active": 0, "command_send": 0, "command_resend": 0, "map_epoch": 1064, "map_full": 0, "map_inc": 106, "osd_sessions": 1, "osd_session_open": 1, "osd_session_close": 0, "osd_laggy": 1, "omap_wr": 0, "omap_rd": 0, "omap_del": 0 }, "objecter-0x55994098f720": { "op_active": 0, "op_laggy": 0, "op_send": 5415951, "op_send_bytes": 205291238, "op_resend": 8, "op_reply": 5415943, "op": 5415943, "op_r": 3612105, "op_w": 1803838, "op_rmw": 0, "op_pg": 0, "osdop_stat": 0, "osdop_create": 0, "osdop_read": 0, "osdop_write": 0, "osdop_writefull": 0, "osdop_writesame": 0, "osdop_append": 0, "osdop_zero": 0, "osdop_truncate": 0, "osdop_delete": 0, "osdop_mapext": 0, "osdop_sparse_read": 0, "osdop_clonerange": 0, "osdop_getxattr": 0, "osdop_setxattr": 0, "osdop_cmpxattr": 0, "osdop_rmxattr": 0, "osdop_resetxattrs": 0, "osdop_call": 5415567, "osdop_watch": 0, "osdop_notify": 0, "osdop_src_cmpxattr": 0, "osdop_pgls": 0, "osdop_pgls_filter": 0, "osdop_other": 376, "linger_active": 0, "linger_send": 0, "linger_resend": 0, "linger_ping": 0, "poolop_active": 0, "poolop_send": 0, "poolop_resend": 0, "poolstat_active": 0, "poolstat_send": 0, "poolstat_resend": 0, "statfs_active": 0, "statfs_send": 0, "statfs_resend": 0, "command_active": 0, "command_send": 0, "command_resend": 0, "map_epoch": 1064, "map_full": 0, "map_inc": 106, "osd_sessions": 8, "osd_session_open": 8834, "osd_session_close": 8826, "osd_laggy": 0, "omap_wr": 0, "omap_rd": 0, "omap_del": 0 }, "rgw": { "req": 2755258, "failed_req": 0, "get": 0, "get_b": 0, "get_initial_lat": { "avgcount": 0, "sum": 0.002219876, "avgtime": 0.000000000 }, "put": 0, "put_b": 0, "put_initial_lat": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 }, "qlen": 0, "qactive": 0, "cache_hit": 0, "cache_miss": 2755261, "keystone_token_cache_hit": 0, "keystone_token_cache_miss": 0, "gc_retire_object": 0, "pubsub_event_triggered": 0, "pubsub_event_lost": 0, "pubsub_store_ok": 0, "pubsub_store_fail": 0, "pubsub_events": 0, "pubsub_push_ok": 0, "pubsub_push_failed": 0, "pubsub_push_pending": 0 }, "simple-throttler": { "throttle": 0 }, "throttle-msgr_dispatch_throttler-radosclient": { "val": 0, "max": 104857600, "get_started": 0, "get": 9379775, "get_sum": 1545393284, "get_or_fail_fail": 0, "get_or_fail_success": 9379775, "take": 0, "take_sum": 0, "put": 9379775, "put_sum": 1545393284, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-radosclient-0x55994098e320": { "val": 0, "max": 104857600, "get_started": 0, "get": 829631, "get_sum": 162850310, "get_or_fail_fail": 0, "get_or_fail_success": 829631, "take": 0, "take_sum": 0, "put": 829631, "put_sum": 162850310, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-msgr_dispatch_throttler-radosclient-0x55994098fa40": { "val": 0, "max": 104857600, "get_started": 0, "get": 5421553, "get_sum": 914508527, "get_or_fail_fail": 0, "get_or_fail_success": 5421553, "take": 0, "take_sum": 0, "put": 5421553, "put_sum": 914508527, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_bytes": { "val": 0, "max": 104857600, "get_started": 0, "get": 2755292, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 2755292, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_bytes-0x55994098e780": { "val": 0, "max": 104857600, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_bytes-0x55994098f7c0": { "val": 0, "max": 104857600, "get_started": 0, "get": 5415614, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 5415614, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_ops": { "val": 0, "max": 24576, "get_started": 0, "get": 2755292, "get_sum": 2755292, "get_or_fail_fail": 0, "get_or_fail_success": 2755292, "take": 0, "take_sum": 0, "put": 2755292, "put_sum": 2755292, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_ops-0x55994098e640": { "val": 0, "max": 24576, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-objecter_ops-0x55994098f0e0": { "val": 0, "max": 24576, "get_started": 0, "get": 5415614, "get_sum": 5415614, "get_or_fail_fail": 0, "get_or_fail_success": 5415614, "take": 0, "take_sum": 0, "put": 5415614, "put_sum": 5415614, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } }, "throttle-rgw_async_rados_ops": { "val": 0, "max": 64, "get_started": 0, "get": 0, "get_sum": 0, "get_or_fail_fail": 0, "get_or_fail_success": 0, "take": 0, "take_sum": 0, "put": 0, "put_sum": 0, "wait": { "avgcount": 0, "sum": 0.000000000, "avgtime": 0.000000000 } } } ` var osdRawSchema = ` { "osd": { "op_wip": { "type": 2, "metric_type": "gauge", "value_type": "integer", "description": "Replication operations currently being processed (primary)", "nick": "", "priority": 5, "units": "none" }, "op": { "type": 10, "metric_type": "counter", "value_type": "integer", "description": "Client operations", "nick": "ops", "priority": 10, "units": "none" }, "op_in_bytes": { "type": 10, "metric_type": "counter", "value_type": "integer", "description": "Client operations total write size", "nick": "wr", "priority": 8, "units": "bytes" }, "op_out_bytes": { "type": 10, "metric_type": "counter", "value_type": "integer", "description": "Client operations total read size", "nick": "rd", "priority": 8, "units": "bytes" }, "op_latency": { "type": 5, "metric_type": "gauge", "value_type": "real-integer-pair", "description": "Latency of client operations (including queue time)", "nick": "l", "priority": 9, "units": "none" }, "op_process_latency": { "type": 5, "metric_type": "gauge", "value_type": "real-integer-pair", "description": "Latency of client operations (excluding queue time)", "nick": "", "priority": 5, "units": "none" }, "op_prepare_latency": { "type": 5, "metric_type": "gauge", "value_type": "real-integer-pair", "description": "Latency of client operations (excluding queue time and wait for finished)", "nick": "", "priority": 5, "units": "none" } } } ` var clusterStatusDump = ` { "health": { "status": "HEALTH_OK", "checks": {}, "mutes": [] }, "fsid": "01234567-abcd-9876-0123-ffeeddccbbaa", "election_epoch": 504, "quorum": [ 0, 1, 2 ], "quorum_names": [ "a", "b", "c" ], "monmap": { "epoch": 27, "min_mon_release_name": "pacific", "num_mons": 3 }, "osdmap": { "epoch": 21734, "num_osds": 24, "num_up_osds": 24, "osd_up_since": 1646008974, "num_in_osds": 24, "osd_in_since": 1637261831, "num_remapped_pgs": 0 }, "pgmap": { "pgs_by_state": [ { "state_name": "active+clean", "count": 2560 }, { "state_name": "active+scrubbing", "count": 10 }, { "state_name": "active+backfilling", "count": 5 } ], "version": 52314277, "num_pgs": 2560, "num_pools": 4, "num_objects": 974827, "data_bytes": 2700031960713, "bytes_used": 7478347665408, "bytes_avail": 9857462382592, "bytes_total": 17335810048000, "inactive_pgs_ratio": 0.19742488861083984, "degraded_objects": 25919, "degraded_total": 2920050, "degraded_ratio": 0.008876217872981627, "recovering_objects_per_sec": 6, "recovering_bytes_per_sec": 223303, "recovering_keys_per_sec": 10, "num_objects_recovered": 40, "num_bytes_recovered": 1335808, "num_keys_recovered": 64, "read_bytes_sec": 0, "write_bytes_sec": 367217, "read_op_per_sec": 322, "write_op_per_sec": 1022 }, "mdsmap": { "epoch": 1, "up": 0, "in": 0, "max": 0, "by_rank": [] }, "fsmap": { "epoch": 24756, "id": 4, "up": 1, "in": 1, "max": 1, "by_rank": [ { "filesystem_id": 4, "rank": 0, "name": "a", "status": "up:active", "gid": 83585495 } ], "up:standby": 2 }, "mgrmap": { "available": true, "num_standbys": 2, "modules": [ "dashboard", "iostat", "restful", "stats" ], "services": { "dashboard": "https://192.168.0.1:8443/" } }, "servicemap": { "epoch": 219919, "modified": "2022-03-07T13:52:42.430153-0500", "services": {} }, "progress_events": {} } ` var cephStatusResults = []expectedResult{ { metric: "ceph_fsmap", fields: map[string]interface{}{ "in": float64(1), "max": float64(1), "up_standby": float64(2), "up": float64(1), }, tags: map[string]string{}, }, { metric: "ceph_health", fields: map[string]interface{}{ "overall_status": "", "status_code": float64(2), "status": "HEALTH_OK", }, tags: map[string]string{}, }, { metric: "ceph_monmap", fields: map[string]interface{}{ "num_mons": float64(3), }, tags: map[string]string{}, }, { metric: "ceph_osdmap", fields: map[string]interface{}{ "epoch": float64(21734), "num_in_osds": float64(24), "num_osds": float64(24), "num_remapped_pgs": float64(0), "num_up_osds": float64(24), }, tags: map[string]string{}, }, { metric: "ceph_pgmap", fields: map[string]interface{}{ "bytes_avail": float64(9857462382592), "bytes_total": float64(17335810048000), "bytes_used": float64(7478347665408), "data_bytes": float64(2700031960713), "degraded_objects": float64(25919), "degraded_ratio": float64(0.0088762178729816267), "degraded_total": float64(2920050), "inactive_pgs_ratio": float64(0.19742488861083984), "num_bytes_recovered": float64(1335808), "num_keys_recovered": float64(64), "num_objects_recovered": float64(40), "num_objects": float64(974827), "num_pgs": float64(2560), "num_pools": float64(4), "op_per_sec": float64(0), // exists only in ceph <10 "read_bytes_sec": float64(0), "read_op_per_sec": float64(322), "recovering_bytes_per_sec": float64(223303), "recovering_keys_per_sec": float64(10), "recovering_objects_per_sec": float64(6), "version": float64(52314277), "write_bytes_sec": float64(367217), "write_op_per_sec": float64(1022), }, tags: map[string]string{}, }, { metric: "ceph_pgmap_state", fields: map[string]interface{}{ "count": float64(2560), }, tags: map[string]string{ "state": "active+clean", }, }, { metric: "ceph_pgmap_state", fields: map[string]interface{}{ "count": float64(10), }, tags: map[string]string{ "state": "active+scrubbing", }, }, { metric: "ceph_pgmap_state", fields: map[string]interface{}{ "count": float64(5), }, tags: map[string]string{ "state": "active+backfilling", }, }, } var clusterStatusDumpNestedOSD = ` { "health": { "status": "HEALTH_OK", "checks": {}, "mutes": [] }, "fsid": "01234567-abcd-9876-0123-ffeeddccbbaa", "election_epoch": 504, "quorum": [ 0, 1, 2 ], "quorum_names": [ "a", "b", "c" ], "monmap": { "epoch": 27, "min_mon_release_name": "pacific", "num_mons": 3 }, "osdmap": { "osdmap": { "epoch": 21734, "num_osds": 24, "num_up_osds": 24, "osd_up_since": 1646008974, "num_in_osds": 24, "osd_in_since": 1637261831, "num_remapped_pgs": 0 } }, "pgmap": { "pgs_by_state": [ { "state_name": "active+clean", "count": 2560 }, { "state_name": "active+scrubbing", "count": 10 }, { "state_name": "active+backfilling", "count": 5 } ], "version": 52314277, "num_pgs": 2560, "num_pools": 4, "num_objects": 974827, "data_bytes": 2700031960713, "bytes_used": 7478347665408, "bytes_avail": 9857462382592, "bytes_total": 17335810048000, "inactive_pgs_ratio": 0.19742488861083984, "degraded_objects": 25919, "degraded_total": 2920050, "degraded_ratio": 0.008876217872981627, "recovering_objects_per_sec": 6, "recovering_bytes_per_sec": 223303, "recovering_keys_per_sec": 10, "num_objects_recovered": 40, "num_bytes_recovered": 1335808, "num_keys_recovered": 64, "read_bytes_sec": 0, "write_bytes_sec": 367217, "read_op_per_sec": 322, "write_op_per_sec": 1022 }, "mdsmap": { "epoch": 1, "up": 0, "in": 0, "max": 0, "by_rank": [] }, "fsmap": { "epoch": 24756, "id": 4, "up": 1, "in": 1, "max": 1, "by_rank": [ { "filesystem_id": 4, "rank": 0, "name": "a", "status": "up:active", "gid": 83585495 } ], "up:standby": 2 }, "mgrmap": { "available": true, "num_standbys": 2, "modules": [ "dashboard", "iostat", "restful", "stats" ], "services": { "dashboard": "https://192.168.0.1:8443/" } }, "servicemap": { "epoch": 219919, "modified": "2022-03-07T13:52:42.430153-0500", "services": {} }, "progress_events": {} } ` var cephStatusResultsNestedOSD = []expectedResult{ { metric: "ceph_osdmap", fields: map[string]interface{}{ "epoch": float64(21734), "full": false, "nearfull": false, "num_in_osds": float64(24), "num_osds": float64(24), "num_remapped_pgs": float64(0), "num_up_osds": float64(24), }, tags: map[string]string{}, }, } var cephDFDump = ` { "stats": { "total_bytes": 14882929901568, "total_avail_bytes": 7866135064576, "total_used_bytes": 7016794836992, "total_used_raw_bytes": 7016794836992, "total_used_raw_ratio": 0.47146594524383545, "num_osds": 6, "num_per_pool_osds": 6, "num_per_pool_omap_osds": 6 }, "stats_by_class": { "hdd": { "total_bytes": 12002349023232, "total_avail_bytes": 6080040206336, "total_used_bytes": 5922308816896, "total_used_raw_bytes": 5922308816896, "total_used_raw_ratio": 0.49342915415763855 }, "ssd": { "total_bytes": 2880580878336, "total_avail_bytes": 1786094858240, "total_used_bytes": 1094486020096, "total_used_raw_bytes": 1094486020096, "total_used_raw_ratio": 0.3799532353878021 } }, "pools": [ { "name": "foo", "id": 4, "stats": { "stored": 672914410122, "objects": 161029, "kb_used": 1972149256, "bytes_used": 2019480838046, "percent_used": 0.2692900002002716, "max_avail": 1826599337984 } }, { "name": "bar_metadata", "id": 12, "stats": { "stored": 1486436928, "objects": 88452, "kb_used": 4354883, "bytes_used": 4459399953, "percent_used": 0.0027088995557278395, "max_avail": 547248046080 } }, { "name": "bar_data", "id": 13, "stats": { "stored": 1291824398336, "objects": 522694, "kb_used": 3800912960, "bytes_used": 3892134871040, "percent_used": 0.41529691219329834, "max_avail": 1826599337984 } }, { "name": "device_health_metrics", "id": 17, "stats": { "stored": 42634584, "objects": 9, "kb_used": 83271, "bytes_used": 85269164, "percent_used": 0.000012548729500849731, "max_avail": 3397479301120 } } ] } ` var cephDfResults = []expectedResult{ { metric: "ceph_usage", fields: map[string]interface{}{ "num_osds": float64(6), "num_per_pool_omap_osds": float64(6), "num_per_pool_osds": float64(6), "total_avail_bytes": float64(7866135064576), "total_avail": float64(0), // pre ceph 0.84 "total_bytes": float64(14882929901568), "total_space": float64(0), // pre ceph 0.84 "total_used_bytes": float64(7016794836992), "total_used_raw_bytes": float64(7016794836992), "total_used_raw_ratio": float64(0.47146594524383545), "total_used": float64(0), // pre ceph 0.84 }, tags: map[string]string{}, }, { metric: "ceph_pool_usage", fields: map[string]interface{}{ "bytes_used": float64(2019480838046), "kb_used": float64(1972149256), "max_avail": float64(1826599337984), "objects": float64(161029), "percent_used": float64(0.26929000020027161), "stored": float64(672914410122), }, tags: map[string]string{ "name": "foo", }, }, { metric: "ceph_pool_usage", fields: map[string]interface{}{ "bytes_used": float64(4459399953), "kb_used": float64(4354883), "max_avail": float64(547248046080), "objects": float64(88452), "percent_used": float64(0.0027088995557278395), "stored": float64(1486436928), }, tags: map[string]string{ "name": "bar_metadata", }, }, { metric: "ceph_pool_usage", fields: map[string]interface{}{ "bytes_used": float64(3892134871040), "kb_used": float64(3800912960), "max_avail": float64(1826599337984), "objects": float64(522694), "percent_used": float64(0.41529691219329834), "stored": float64(1291824398336), }, tags: map[string]string{ "name": "bar_data", }, }, { metric: "ceph_pool_usage", fields: map[string]interface{}{ "bytes_used": float64(85269164), "kb_used": float64(83271), "max_avail": float64(3397479301120), "objects": float64(9), "percent_used": float64(0.000012548729500849731), "stored": float64(42634584), }, tags: map[string]string{ "name": "device_health_metrics", }, }, { metric: "ceph_deviceclass_usage", fields: map[string]interface{}{ "total_avail_bytes": float64(6080040206336), "total_bytes": float64(12002349023232), "total_used_bytes": float64(5922308816896), "total_used_raw_bytes": float64(5922308816896), "total_used_raw_ratio": float64(0.49342915415763855), }, tags: map[string]string{ "class": "hdd", }, }, { metric: "ceph_deviceclass_usage", fields: map[string]interface{}{ "total_avail_bytes": float64(1786094858240), "total_bytes": float64(2880580878336), "total_used_bytes": float64(1094486020096), "total_used_raw_bytes": float64(1094486020096), "total_used_raw_ratio": float64(0.3799532353878021), }, tags: map[string]string{ "class": "ssd", }, }, } var cephODSPoolStatsDump = ` [ { "pool_name": "data", "pool_id": 0, "recovery": { "degraded_objects": 12, "degraded_total": 483087, "degraded_ratio": 0.000024840246166839513 }, "recovery_rate": {}, "client_io_rate": {} }, { "pool_name": "metadata", "pool_id": 1, "recovery": {}, "recovery_rate": {}, "client_io_rate": {} }, { "pool_name": "rbd", "pool_id": 2, "recovery": {}, "recovery_rate": {}, "client_io_rate": {} }, { "pool_name": "pbench", "pool_id": 3, "recovery": {}, "recovery_rate": { "recovering_objects_per_sec": 16, "recovering_bytes_per_sec": 0, "recovering_keys_per_sec": 0, "num_objects_recovered": 200, "num_bytes_recovered": 0, "num_keys_recovered": 0 }, "client_io_rate": { "read_bytes_sec": 10566067, "write_bytes_sec": 15165220376, "read_op_per_sec": 182, "write_op_per_sec": 473 } } ] ` var cephOSDPoolStatsResults = []expectedResult{ { metric: "ceph_pool_stats", fields: map[string]interface{}{ "degraded_objects": float64(12), "degraded_ratio": float64(0.000024840246166839513), "degraded_total": float64(483087), "num_bytes_recovered": float64(0), "num_keys_recovered": float64(0), "num_objects_recovered": float64(0), "op_per_sec": float64(0), // This field is no longer reported in ceph 10 and later "read_bytes_sec": float64(0), "read_op_per_sec": float64(0), "recovering_bytes_per_sec": float64(0), "recovering_keys_per_sec": float64(0), "recovering_objects_per_sec": float64(0), "write_bytes_sec": float64(0), "write_op_per_sec": float64(0), }, tags: map[string]string{ "name": "data", }, }, { metric: "ceph_pool_stats", fields: map[string]interface{}{ "degraded_objects": float64(0), "degraded_ratio": float64(0), "degraded_total": float64(0), "num_bytes_recovered": float64(0), "num_keys_recovered": float64(0), "num_objects_recovered": float64(0), "op_per_sec": float64(0), // This field is no longer reported in ceph 10 and later "read_bytes_sec": float64(0), "read_op_per_sec": float64(0), "recovering_bytes_per_sec": float64(0), "recovering_keys_per_sec": float64(0), "recovering_objects_per_sec": float64(0), "write_bytes_sec": float64(0), "write_op_per_sec": float64(0), }, tags: map[string]string{ "name": "metadata", }, }, { metric: "ceph_pool_stats", fields: map[string]interface{}{ "degraded_objects": float64(0), "degraded_ratio": float64(0), "degraded_total": float64(0), "num_bytes_recovered": float64(0), "num_keys_recovered": float64(0), "num_objects_recovered": float64(200), "op_per_sec": float64(0), // This field is no longer reported in ceph 10 and later "read_bytes_sec": float64(10566067), "read_op_per_sec": float64(182), "recovering_bytes_per_sec": float64(0), "recovering_keys_per_sec": float64(0), "recovering_objects_per_sec": float64(16), "write_bytes_sec": float64(15165220376), "write_op_per_sec": float64(473), }, tags: map[string]string{ "name": "pbench", }, }, }