Report inode space usage statistics to etcd, aggregate it in the monitor

rdma-zerocopy
Vitaliy Filippov 2021-01-19 02:14:13 +03:00
parent c35963967f
commit 4ae1b84c67
5 changed files with 51 additions and 8 deletions

View File

@ -26,11 +26,13 @@ const etcd_allow = new RegExp('^'+[
'config/pgs',
'osd/state/[1-9]\\d*',
'osd/stats/[1-9]\\d*',
'osd/space/[1-9]\\d*',
'mon/master',
'pg/state/[1-9]\\d*/[1-9]\\d*',
'pg/stats/[1-9]\\d*/[1-9]\\d*',
'pg/history/[1-9]\\d*/[1-9]\\d*',
'history/last_clean_pgs',
'inode/space/[1-9]\\d*',
'stats',
].join('$|^')+'$');
@ -172,6 +174,11 @@ const etcd_tree = {
},
}, */
},
space: {
/* <osd_num_t>: {
<inode_t>: uint64_t, // bytes
}, */
},
},
mon: {
master: {
@ -211,6 +218,13 @@ const etcd_tree = {
}, */
},
},
inode: {
space: {
/* <inode_t>: {
raw: uint64_t, // raw bytes on OSDs
}, */
},
},
stats: {
/* op_stats: {
<string>: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
@ -403,7 +417,7 @@ class Mon
{
pg_states_changed = true;
}
else if (key != '/stats')
else if (key != '/stats' && key.substr(0, 13) != '/inode/space/')
{
changed = true;
}
@ -1174,6 +1188,7 @@ class Mon
async update_total_stats()
{
const txn = [];
const stats = this.sum_stats();
if (!stats.overflow)
{
@ -1196,9 +1211,26 @@ class Mon
{
ser.object_counts[k] = ''+stats.object_counts[k];
}
await this.etcd_call('/kv/txn', {
success: [ { requestPut: { key: b64(this.etcd_prefix+'/stats'), value: b64(JSON.stringify(ser)) } } ],
}, this.config.etcd_mon_timeout, 0);
txn.push({ requestPut: { key: b64(this.etcd_prefix+'/stats'), value: b64(JSON.stringify(ser)) } });
}
const space_stats = {};
for (const osd_num in this.state.osd.space)
{
for (const inode_num in this.state.osd.space[osd_num])
{
space_stats[inode_num] = (space_stats[inode_num] || BigInt(0)) + BigInt(this.state.osd.space[osd_num][inode_num]||0);
}
}
for (const inode_num in space_stats)
{
txn.push({ requestPut: {
key: b64(this.etcd_prefix+'/inode/space/'+inode_num),
value: b64(JSON.stringify({ raw: ''+space_stats[inode_num] })),
} });
}
if (txn.length)
{
await this.etcd_call('/kv/txn', { success: txn }, this.config.etcd_mon_timeout, 0);
}
}

View File

@ -43,7 +43,7 @@ std::unordered_map<object_id, uint64_t> & blockstore_t::get_unstable_writes()
return impl->unstable_writes;
}
std::map<uint64_t, int64_t> & blockstore_t::get_inode_space_stats()
std::map<uint64_t, uint64_t> & blockstore_t::get_inode_space_stats()
{
return impl->inode_space_stats;
}

View File

@ -184,7 +184,7 @@ public:
std::unordered_map<object_id, uint64_t> & get_unstable_writes();
// Get per-inode space usage statistics
std::map<uint64_t, int64_t> & get_inode_space_stats();
std::map<uint64_t, uint64_t> & get_inode_space_stats();
// FIXME rename to object_size
uint32_t get_block_size();

View File

@ -328,7 +328,7 @@ public:
std::unordered_map<object_id, uint64_t> unstable_writes;
// Space usage statistics
std::map<uint64_t, int64_t> inode_space_stats;
std::map<uint64_t, uint64_t> inode_space_stats;
inline uint32_t get_block_size() { return block_size; }
inline uint64_t get_block_count() { return block_count; }

View File

@ -180,11 +180,22 @@ void osd_t::report_statistics()
return;
}
etcd_reporting_stats = true;
// Report space usage statistics as a whole
// Maybe we'll report it using deltas if we tune for a lot of inodes at some point
json11::Json::object inode_space;
for (auto kv: bs->get_inode_space_stats())
{
inode_space[std::to_string(kv.first)] = kv.second;
}
json11::Json::array txn = { json11::Json::object {
{ "request_put", json11::Json::object {
{ "key", base64_encode(st_cli.etcd_prefix+"/osd/stats/"+std::to_string(osd_num)) },
{ "value", base64_encode(get_statistics().dump()) },
} }
} },
{ "request_put", json11::Json::object {
{ "key", base64_encode(st_cli.etcd_prefix+"/osd/space/"+std::to_string(osd_num)) },
{ "value", base64_encode(json11::Json(inode_space).dump()) },
} },
} };
for (auto & p: pgs)
{