diff --git a/mon/mon.js b/mon/mon.js index 5fd202b65..4f5ada393 100644 --- a/mon/mon.js +++ b/mon/mon.js @@ -26,11 +26,13 @@ const etcd_allow = new RegExp('^'+[ 'config/pgs', 'osd/state/[1-9]\\d*', 'osd/stats/[1-9]\\d*', + 'osd/space/[1-9]\\d*', 'mon/master', 'pg/state/[1-9]\\d*/[1-9]\\d*', 'pg/stats/[1-9]\\d*/[1-9]\\d*', 'pg/history/[1-9]\\d*/[1-9]\\d*', 'history/last_clean_pgs', + 'inode/space/[1-9]\\d*', 'stats', ].join('$|^')+'$'); @@ -172,6 +174,11 @@ const etcd_tree = { }, }, */ }, + space: { + /* : { + : uint64_t, // bytes + }, */ + }, }, mon: { master: { @@ -211,6 +218,13 @@ const etcd_tree = { }, */ }, }, + inode: { + space: { + /* : { + raw: uint64_t, // raw bytes on OSDs + }, */ + }, + }, stats: { /* op_stats: { : { count: uint64_t, usec: uint64_t, bytes: uint64_t }, @@ -403,7 +417,7 @@ class Mon { pg_states_changed = true; } - else if (key != '/stats') + else if (key != '/stats' && key.substr(0, 13) != '/inode/space/') { changed = true; } @@ -1174,6 +1188,7 @@ class Mon async update_total_stats() { + const txn = []; const stats = this.sum_stats(); if (!stats.overflow) { @@ -1196,9 +1211,26 @@ class Mon { ser.object_counts[k] = ''+stats.object_counts[k]; } - await this.etcd_call('/kv/txn', { - success: [ { requestPut: { key: b64(this.etcd_prefix+'/stats'), value: b64(JSON.stringify(ser)) } } ], - }, this.config.etcd_mon_timeout, 0); + txn.push({ requestPut: { key: b64(this.etcd_prefix+'/stats'), value: b64(JSON.stringify(ser)) } }); + } + const space_stats = {}; + for (const osd_num in this.state.osd.space) + { + for (const inode_num in this.state.osd.space[osd_num]) + { + space_stats[inode_num] = (space_stats[inode_num] || BigInt(0)) + BigInt(this.state.osd.space[osd_num][inode_num]||0); + } + } + for (const inode_num in space_stats) + { + txn.push({ requestPut: { + key: b64(this.etcd_prefix+'/inode/space/'+inode_num), + value: b64(JSON.stringify({ raw: ''+space_stats[inode_num] })), + } }); + } + if (txn.length) + { + await this.etcd_call('/kv/txn', { success: txn }, this.config.etcd_mon_timeout, 0); } } diff --git a/src/blockstore.cpp b/src/blockstore.cpp index 783db4fa8..a73ed34bd 100644 --- a/src/blockstore.cpp +++ b/src/blockstore.cpp @@ -43,7 +43,7 @@ std::unordered_map & blockstore_t::get_unstable_writes() return impl->unstable_writes; } -std::map & blockstore_t::get_inode_space_stats() +std::map & blockstore_t::get_inode_space_stats() { return impl->inode_space_stats; } diff --git a/src/blockstore.h b/src/blockstore.h index bc4bc29ae..0cf43a372 100644 --- a/src/blockstore.h +++ b/src/blockstore.h @@ -184,7 +184,7 @@ public: std::unordered_map & get_unstable_writes(); // Get per-inode space usage statistics - std::map & get_inode_space_stats(); + std::map & get_inode_space_stats(); // FIXME rename to object_size uint32_t get_block_size(); diff --git a/src/blockstore_impl.h b/src/blockstore_impl.h index 1376cb28c..9e9b56066 100644 --- a/src/blockstore_impl.h +++ b/src/blockstore_impl.h @@ -328,7 +328,7 @@ public: std::unordered_map unstable_writes; // Space usage statistics - std::map inode_space_stats; + std::map inode_space_stats; inline uint32_t get_block_size() { return block_size; } inline uint64_t get_block_count() { return block_count; } diff --git a/src/osd_cluster.cpp b/src/osd_cluster.cpp index e5355db8e..12c59ee97 100644 --- a/src/osd_cluster.cpp +++ b/src/osd_cluster.cpp @@ -180,11 +180,22 @@ void osd_t::report_statistics() return; } etcd_reporting_stats = true; + // Report space usage statistics as a whole + // Maybe we'll report it using deltas if we tune for a lot of inodes at some point + json11::Json::object inode_space; + for (auto kv: bs->get_inode_space_stats()) + { + inode_space[std::to_string(kv.first)] = kv.second; + } json11::Json::array txn = { json11::Json::object { { "request_put", json11::Json::object { { "key", base64_encode(st_cli.etcd_prefix+"/osd/stats/"+std::to_string(osd_num)) }, { "value", base64_encode(get_statistics().dump()) }, - } } + } }, + { "request_put", json11::Json::object { + { "key", base64_encode(st_cli.etcd_prefix+"/osd/space/"+std::to_string(osd_num)) }, + { "value", base64_encode(json11::Json(inode_space).dump()) }, + } }, } }; for (auto & p: pgs) {