huynp911
/
vitastor
forked from vitalif/vitastor
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Calculate average statistics in mon, remove buggy "fix_stat_overflows"

nbd-vmsplice
Vitaliy Filippov 2021-11-10 23:51:15 +03:00
parent 0544a16f95
commit 0f3f0a9d29
1 changed files with 58 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
mon/mon.js
View File

@ -265,9 +265,9 @@ const etcd_tree = {
/* <pool_id>: { /* <pool_id>: {
<inode_t>: { <inode_t>: {
raw_used: uint64_t, // raw used bytes on OSDs raw_used: uint64_t, // raw used bytes on OSDs
read: { count: uint64_t, usec: uint64_t, bytes: uint64_t }, read: { count: uint64_t, usec: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t, lat: uint64_t },
write: { count: uint64_t, usec: uint64_t, bytes: uint64_t }, write: { count: uint64_t, usec: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t, lat: uint64_t },
delete: { count: uint64_t, usec: uint64_t, bytes: uint64_t }, delete: { count: uint64_t, usec: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t, lat: uint64_t },
}, },
}, */ }, */
}, },
@ -284,14 +284,14 @@ const etcd_tree = {
}, },
stats: { stats: {
/* op_stats: { /* op_stats: {
<string>: { count: uint64_t, usec: uint64_t, bytes: uint64_t }, <string>: { count: uint64_t, usec: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t, lat: uint64_t },
}, },
subop_stats: { subop_stats: {
<string>: { count: uint64_t, usec: uint64_t }, <string>: { count: uint64_t, usec: uint64_t, iops: uint64_t, lat: uint64_t },
}, },
recovery_stats: { recovery_stats: {
degraded: { count: uint64_t, bytes: uint64_t }, degraded: { count: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t },
misplaced: { count: uint64_t, bytes: uint64_t }, misplaced: { count: uint64_t, bytes: uint64_t, bps: uint64_t, iops: uint64_t },
}, },
object_counts: { object_counts: {
object: uint64_t, object: uint64_t,
@ -1198,7 +1198,7 @@ class Mon
}, this.config.mon_change_timeout || 1000); }, this.config.mon_change_timeout || 1000);
} }
sum_op_stats() sum_op_stats(timestamp, prev_stats)
{ {
const op_stats = {}, subop_stats = {}, recovery_stats = {}; const op_stats = {}, subop_stats = {}, recovery_stats = {};
for (const osd in this.state.osd.stats) for (const osd in this.state.osd.stats)
@ -1224,6 +1224,29 @@ class Mon
recovery_stats[op].bytes += BigInt(st.recovery_stats[op].bytes||0); recovery_stats[op].bytes += BigInt(st.recovery_stats[op].bytes||0);
} }
} }
if (prev_stats && prev_stats.timestamp >= timestamp)
{
prev_stats = null;
}
const tm = prev_stats ? BigInt(timestamp - prev_stats.timestamp) : 0;
for (const op in op_stats)
{
op_stats[op].bps = prev_stats ? (op_stats[op].bytes - prev_stats.op_stats[op].bytes) * 1000n / tm : 0;
op_stats[op].iops = prev_stats ? (op_stats[op].count - prev_stats.op_stats[op].count) * 1000n / tm : 0;
op_stats[op].lat = prev_stats ? (op_stats[op].usec - prev_stats.op_stats[op].usec)
/ ((op_stats[op].count - prev_stats.op_stats[op].count) || 1n) : 0;
}
for (const op in subop_stats)
{
subop_stats[op].iops = prev_stats ? (subop_stats[op].count - prev_stats.subop_stats[op].count) * 1000n / tm : 0;
subop_stats[op].lat = prev_stats ? (subop_stats[op].usec - prev_stats.subop_stats[op].usec)
/ ((subop_stats[op].count - prev_stats.subop_stats[op].count) || 1n) : 0;
}
for (const op in recovery_stats)
{
recovery_stats[op].bps = prev_stats ? (recovery_stats[op].bytes - prev_stats.recovery_stats[op].bytes) * 1000n / tm : 0;
recovery_stats[op].iops = prev_stats ? (recovery_stats[op].count - prev_stats.recovery_stats[op].count) * 1000n / tm : 0;
}
return { op_stats, subop_stats, recovery_stats }; return { op_stats, subop_stats, recovery_stats };
} }
@ -1250,7 +1273,7 @@ class Mon
return object_counts; return object_counts;
} }
sum_inode_stats() sum_inode_stats(prev_stats, timestamp, prev_timestamp)
{ {
const inode_stats = {}; const inode_stats = {};
const inode_stub = () => ({ const inode_stub = () => ({
@ -1309,43 +1332,31 @@ class Mon
} }
} }
} }
return inode_stats; if (prev_stats && prev_timestamp >= timestamp)
}
fix_stat_overflows(obj, scratch)
{
for (const k in obj)
{ {
if (typeof obj[k] == 'bigint') prev_stats = null;
}
const tm = prev_stats ? BigInt(timestamp - prev_timestamp) : 0;
for (const pool_id in inode_stats)
{
for (const inode_num in inode_stats[pool_id])
{ {
if (obj[k] >= 0x10000000000000000n) for (const op of [ 'read', 'write', 'delete' ])
{ {
if (scratch[k]) const op_st = inode_stats[pool_id][inode_num][op];
{ const prev_st = prev_stats && prev_stats[pool_id] && prev_stats[pool_id][inode_num] && prev_stats[pool_id][inode_num][op];
for (const k2 in scratch) op_st.bps = prev_st ? (op_st.bytes - prev_st.bytes) * 1000n / tm : 0;
{ op_st.iops = prev_st ? (op_st.count - prev_st.count) * 1000n / tm : 0;
obj[k2] -= scratch[k2]; op_st.lat = prev_st ? (op_st.usec - prev_st.usec) / ((op_st.count - prev_st.count) || 1n) : 0;
scratch[k2] = 0n;
}
}
else
{
for (const k2 in obj)
{
scratch[k2] = obj[k2];
}
}
} }
} }
else if (typeof obj[k] == 'object')
{
this.fix_stat_overflows(obj[k], scratch[k] = (scratch[k] || {}));
}
} }
return inode_stats;
} }
serialize_bigints(obj) serialize_bigints(obj)
{ {
obj = { ...obj };
for (const k in obj) for (const k in obj)
{ {
if (typeof obj[k] == 'bigint') if (typeof obj[k] == 'bigint')
@ -1354,22 +1365,26 @@ class Mon
} }
else if (typeof obj[k] == 'object') else if (typeof obj[k] == 'object')
{ {
this.serialize_bigints(obj[k]); obj[k] = this.serialize_bigints(obj[k]);
} }
} }
return obj;
} }
async update_total_stats() async update_total_stats()
{ {
const txn = []; const txn = [];
const stats = this.sum_op_stats(); const timestamp = Date.now();
const object_counts = this.sum_object_counts(); const object_counts = this.sum_object_counts();
const inode_stats = this.sum_inode_stats(); let stats = this.sum_op_stats(timestamp, this.prev_stats);
this.fix_stat_overflows(stats, (this.prev_stats = this.prev_stats || {})); let inode_stats = this.sum_inode_stats(
this.fix_stat_overflows(inode_stats, (this.prev_inode_stats = this.prev_inode_stats || {})); this.prev_stats ? this.prev_stats.inode_stats : null,
timestamp, this.prev_stats ? this.prev_stats.timestamp : null
);
this.prev_stats = { timestamp, ...stats, inode_stats };
stats.object_counts = object_counts; stats.object_counts = object_counts;
this.serialize_bigints(stats); stats = this.serialize_bigints(stats);
this.serialize_bigints(inode_stats); inode_stats = this.serialize_bigints(inode_stats);
txn.push({ requestPut: { key: b64(this.etcd_prefix+'/stats'), value: b64(JSON.stringify(stats)) } }); txn.push({ requestPut: { key: b64(this.etcd_prefix+'/stats'), value: b64(JSON.stringify(stats)) } });
for (const pool_id in inode_stats) for (const pool_id in inode_stats)
{ {