Report inode space usage statistics to etcd, aggregate it in the monitor

mon/mon.js

@@ -26,11 +26,13 @@ const etcd_allow = new RegExp('^'+[
     'config/pgs',
     'osd/state/[1-9]\\d*',
     'osd/stats/[1-9]\\d*',
+    'osd/space/[1-9]\\d*',
     'mon/master',
     'pg/state/[1-9]\\d*/[1-9]\\d*',
     'pg/stats/[1-9]\\d*/[1-9]\\d*',
     'pg/history/[1-9]\\d*/[1-9]\\d*',
     'history/last_clean_pgs',
+    'inode/space/[1-9]\\d*',
     'stats',
 ].join('$|^')+'$');
 
@@ -172,6 +174,11 @@ const etcd_tree = {
                 },
             }, */
         },
+        space: {
+            /* <osd_num_t>: {
+                <inode_t>: uint64_t, // bytes
+            }, */
+        },
     },
     mon: {
         master: {
@@ -211,6 +218,13 @@ const etcd_tree = {
             }, */
         },
     },
+    inode: {
+        space: {
+            /* <inode_t>: {
+                raw: uint64_t, // raw bytes on OSDs
+            }, */
+        },
+    },
     stats: {
         /* op_stats: {
             <string>: { count: uint64_t, usec: uint64_t, bytes: uint64_t },
@@ -403,7 +417,7 @@ class Mon
                     {
                         pg_states_changed = true;
                     }
-                    else if (key != '/stats')
+                    else if (key != '/stats' && key.substr(0, 13) != '/inode/space/')
                     {
                         changed = true;
                     }
@@ -1174,6 +1188,7 @@ class Mon
 
     async update_total_stats()
     {
+        const txn = [];
         const stats = this.sum_stats();
         if (!stats.overflow)
         {
@@ -1196,9 +1211,26 @@ class Mon
             {
                 ser.object_counts[k] = ''+stats.object_counts[k];
             }
-            await this.etcd_call('/kv/txn', {
-                success: [ { requestPut: { key: b64(this.etcd_prefix+'/stats'), value: b64(JSON.stringify(ser)) } } ],
-            }, this.config.etcd_mon_timeout, 0);
+            txn.push({ requestPut: { key: b64(this.etcd_prefix+'/stats'), value: b64(JSON.stringify(ser)) } });
+        }
+        const space_stats = {};
+        for (const osd_num in this.state.osd.space)
+        {
+            for (const inode_num in this.state.osd.space[osd_num])
+            {
+                space_stats[inode_num] = (space_stats[inode_num] || BigInt(0)) + BigInt(this.state.osd.space[osd_num][inode_num]||0);
+            }
+        }
+        for (const inode_num in space_stats)
+        {
+            txn.push({ requestPut: {
+                key: b64(this.etcd_prefix+'/inode/space/'+inode_num),
+                value: b64(JSON.stringify({ raw: ''+space_stats[inode_num] })),
+            } });
+        }
+        if (txn.length)
+        {
+            await this.etcd_call('/kv/txn', { success: txn }, this.config.etcd_mon_timeout, 0);
         }
     }
 

src/blockstore.cpp

@@ -43,7 +43,7 @@ std::unordered_map<object_id, uint64_t> & blockstore_t::get_unstable_writes()
     return impl->unstable_writes;
 }
 
-std::map<uint64_t, int64_t> & blockstore_t::get_inode_space_stats()
+std::map<uint64_t, uint64_t> & blockstore_t::get_inode_space_stats()
 {
     return impl->inode_space_stats;
 }

src/blockstore.h

@@ -184,7 +184,7 @@ public:
     std::unordered_map<object_id, uint64_t> & get_unstable_writes();
 
     // Get per-inode space usage statistics
-    std::map<uint64_t, int64_t> & get_inode_space_stats();
+    std::map<uint64_t, uint64_t> & get_inode_space_stats();
 
     // FIXME rename to object_size
     uint32_t get_block_size();

src/blockstore_impl.h

@@ -328,7 +328,7 @@ public:
     std::unordered_map<object_id, uint64_t> unstable_writes;
 
     // Space usage statistics
-    std::map<uint64_t, int64_t> inode_space_stats;
+    std::map<uint64_t, uint64_t> inode_space_stats;
 
     inline uint32_t get_block_size() { return block_size; }
     inline uint64_t get_block_count() { return block_count; }

src/osd_cluster.cpp

@@ -180,11 +180,22 @@ void osd_t::report_statistics()
         return;
     }
     etcd_reporting_stats = true;
+    // Report space usage statistics as a whole
+    // Maybe we'll report it using deltas if we tune for a lot of inodes at some point
+    json11::Json::object inode_space;
+    for (auto kv: bs->get_inode_space_stats())
+    {
+        inode_space[std::to_string(kv.first)] = kv.second;
+    }
     json11::Json::array txn = { json11::Json::object {
         { "request_put", json11::Json::object {
             { "key", base64_encode(st_cli.etcd_prefix+"/osd/stats/"+std::to_string(osd_num)) },
             { "value", base64_encode(get_statistics().dump()) },
-        } }
+        } },
+        { "request_put", json11::Json::object {
+            { "key", base64_encode(st_cli.etcd_prefix+"/osd/space/"+std::to_string(osd_num)) },
+            { "value", base64_encode(json11::Json(inode_space).dump()) },
+        } },
     } };
     for (auto & p: pgs)
     {