vitalif
/
ceph-bench
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Format code

master
Vitaliy Filippov 2022-01-18 01:13:03 +03:00
parent 420debc2f5
commit c0a05c8fea
6 changed files with 429 additions and 421 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Makefile
View File

@ -16,7 +16,7 @@ clean:
$(CC) $(CPPFLAGS) $(CXXFLAGS) -c $< -o $@ $(CC) $(CPPFLAGS) $(CXXFLAGS) -c $< -o $@
indent: *.cpp *.h indent: *.cpp *.h
clang-format-6.0 -i $^ clang-format -style=Microsoft -i $^
builddep: builddep:
sudo apt install -y --no-install-recommends libjsoncpp-dev libtcmalloc-minimal4 sudo apt install -y --no-install-recommends libjsoncpp-dev libtcmalloc-minimal4

635
main.cpp
View File

@ -1,14 +1,14 @@
#include <chrono> #include <chrono>
#include <csignal> #include <csignal>
#include <fstream>
#include <iostream> #include <iostream>
#include <librados.hpp> #include <librados.hpp>
#include <map> #include <map>
#include <set> #include <set>
#include <string> #include <string>
#include <fstream> #include <system_error>
#include <thread> #include <thread>
#include <vector> #include <vector>
#include <system_error>
#include "mysignals.h" #include "mysignals.h"
#include "radosutil.h" #include "radosutil.h"
@ -19,399 +19,388 @@ using namespace chrono;
struct bench_settings struct bench_settings
{ {
string pool; string pool;
string mode; string mode;
string specific_bench_item; string specific_bench_item;
int threads; int threads;
int secs; int secs;
size_t object_size; size_t object_size;
size_t block_size; size_t block_size;
}; };
template <class T> static double dur2sec(const T &dur) template <class T> static double dur2sec(const T &dur)
{ {
return duration_cast<duration<double>>(dur).count(); return duration_cast<duration<double>>(dur).count();
} }
template <class T> static double dur2msec(const T &dur) template <class T> static double dur2msec(const T &dur)
{ {
return duration_cast<duration<double, milli>>(dur).count(); return duration_cast<duration<double, milli>>(dur).count();
} }
template <class T> static uint64_t dur2nsec(const T &dur) template <class T> static uint64_t dur2nsec(const T &dur)
{ {
return duration_cast<duration<uint64_t, nano>>(dur).count(); return duration_cast<duration<uint64_t, nano>>(dur).count();
} }
template <class T> template <class T> static void print_breakdown(const vector<T> &all_ops, size_t thread_count)
static void print_breakdown(const vector<T> &all_ops, size_t thread_count)
{ {
T totaltime(0); T totaltime(0);
map<size_t, size_t> dur2count; map<size_t, size_t> dur2count;
map<size_t, T> dur2totaltime; map<size_t, T> dur2totaltime;
T mindur(minutes(42)); T mindur(minutes(42));
T maxdur(0); T maxdur(0);
size_t maxcount = 0; size_t maxcount = 0;
for (const auto &res : all_ops) for (const auto &res : all_ops)
{ {
totaltime += res; totaltime += res;
if (res > maxdur) if (res > maxdur)
maxdur = res; maxdur = res;
if (res < mindur) if (res < mindur)
mindur = res; mindur = res;
const auto nsec = dur2nsec(res); const auto nsec = dur2nsec(res);
size_t baserange = 10; size_t baserange = 10;
while (nsec >= baserange) while (nsec >= baserange)
baserange *= 10; baserange *= 10;
baserange /= 10; baserange /= 10;
const size_t range = (nsec / baserange) * baserange; const size_t range = (nsec / baserange) * baserange;
const auto cnt = ++(dur2count[range]); const auto cnt = ++(dur2count[range]);
if (cnt > maxcount) if (cnt > maxcount)
maxcount = cnt; maxcount = cnt;
dur2totaltime[range] += res; dur2totaltime[range] += res;
} }
cout << "min latency " << dur2msec(mindur) << " ms" << endl; cout << "min latency " << dur2msec(mindur) << " ms" << endl;
cout << "max latency " << dur2msec(maxdur) << " ms" << endl; cout << "max latency " << dur2msec(maxdur) << " ms" << endl;
const size_t maxbarsize = 30; const size_t maxbarsize = 30;
for (const auto &p : dur2count) for (const auto &p : dur2count)
{ {
const auto &nsecgrp = p.first; const auto &nsecgrp = p.first;
const auto &count = p.second; const auto &count = p.second;
const auto barsize = count * maxbarsize / maxcount; const auto barsize = count * maxbarsize / maxcount;
auto bar = string(barsize, '#') + string(maxbarsize - barsize, ' '); auto bar = string(barsize, '#') + string(maxbarsize - barsize, ' ');
cout << ">=" << setw(5) << nsecgrp / 1000000.0; cout << ">=" << setw(5) << nsecgrp / 1000000.0;
cout << " ms: " << setw(3) << count * 100 / all_ops.size() << "% " << bar; cout << " ms: " << setw(3) << count * 100 / all_ops.size() << "% " << bar;
cout << " cnt=" << count << endl; cout << " cnt=" << count << endl;
} }
cout << "Average iops: " << (all_ops.size() * thread_count / dur2sec(totaltime)) << endl; cout << "Average iops: " << (all_ops.size() * thread_count / dur2sec(totaltime)) << endl;
cout << "Average latency: " << (dur2msec(totaltime) / all_ops.size()) << " ms" << endl; cout << "Average latency: " << (dur2msec(totaltime) / all_ops.size()) << " ms" << endl;
cout << "Total writes: " << all_ops.size() << endl; cout << "Total writes: " << all_ops.size() << endl;
if (thread_count > 1) if (thread_count > 1)
cout << "iops per thread: " << (all_ops.size() / dur2sec(totaltime)) << endl; cout << "iops per thread: " << (all_ops.size() / dur2sec(totaltime)) << endl;
} }
static void fill_urandom(char *buf, size_t len) static void fill_urandom(char *buf, size_t len)
{ {
ifstream infile; ifstream infile;
infile.exceptions(ifstream::failbit | ifstream::badbit); infile.exceptions(ifstream::failbit | ifstream::badbit);
infile.open("/dev/urandom", ios::binary | ios::in); infile.open("/dev/urandom", ios::binary | ios::in);
infile.read(buf, len); infile.read(buf, len);
} }
// May be called in a thread. // May be called in a thread.
static void _do_bench( static void _do_bench(const unique_ptr<bench_settings> &settings, const vector<string> &obj_names, IoCtx &ioctx,
const unique_ptr<bench_settings> &settings, vector<steady_clock::duration> &ops)
const vector<string> &obj_names,
IoCtx &ioctx,
vector<steady_clock::duration> &ops)
{ {
// TODO: pass bufferlist as arguments // TODO: pass bufferlist as arguments
bufferlist bar1; bufferlist bar1;
bufferlist bar2; bufferlist bar2;
bar1.append(ceph::buffer::create(settings->block_size)); bar1.append(ceph::buffer::create(settings->block_size));
fill_urandom(bar1.c_str(), settings->block_size); fill_urandom(bar1.c_str(), settings->block_size);
bar2.append(ceph::buffer::create(settings->block_size)); bar2.append(ceph::buffer::create(settings->block_size));
fill_urandom(bar2.c_str(), settings->block_size); fill_urandom(bar2.c_str(), settings->block_size);
if (bar1.contents_equal(bar2)) if (bar1.contents_equal(bar2))
throw "Your RNG is not random"; throw "Your RNG is not random";
auto b = steady_clock::now(); auto b = steady_clock::now();
const auto stop = b + seconds(settings->secs); const auto stop = b + seconds(settings->secs);
for (const auto &obj_name : obj_names) for (const auto &obj_name : obj_names)
{
ioctx.remove(obj_name);
}
while (b <= stop)
{
abort_if_signalled();
if (ioctx.write(
obj_names[rand() % 16],
(ops.size() % 2) ? bar1 : bar2,
settings->block_size,
settings->block_size * (rand() % (settings->object_size / settings->block_size))
) < 0)
{ {
throw "Write error"; ioctx.remove(obj_name);
}
while (b <= stop)
{
abort_if_signalled();
if (ioctx.write(obj_names[rand() % 16], (ops.size() % 2) ? bar1 : bar2, settings->block_size,
settings->block_size * (rand() % (settings->object_size / settings->block_size))) < 0)
{
throw "Write error";
}
const auto b2 = steady_clock::now();
ops.push_back(b2 - b);
b = b2;
} }
const auto b2 = steady_clock::now();
ops.push_back(b2 - b);
b = b2;
}
} }
static void do_bench(const unique_ptr<bench_settings> &settings, const vector<string> &names, IoCtx &ioctx) static void do_bench(const unique_ptr<bench_settings> &settings, const vector<string> &names, IoCtx &ioctx)
{ {
vector<steady_clock::duration> all_ops; vector<steady_clock::duration> all_ops;
if (settings->threads > 1) if (settings->threads > 1)
{
vector<thread> threads;
vector<vector<steady_clock::duration>> listofops;
for (int i = 0; i < settings->threads; i++)
{ {
listofops.push_back(vector<steady_clock::duration>()); vector<thread> threads;
} vector<vector<steady_clock::duration>> listofops;
for (int i = 0; i < settings->threads; i++) for (int i = 0; i < settings->threads; i++)
{
listofops.push_back(vector<steady_clock::duration>());
}
for (int i = 0; i < settings->threads; i++)
{
sigset_t new_set;
sigset_t old_set;
sigfillset(&new_set);
int err;
if ((err = pthread_sigmask(SIG_SETMASK, &new_set, &old_set)))
{
throw std::system_error(err, std::system_category(), "Failed to set thread sigmask");
}
threads.push_back(thread(_do_bench, ref(settings),
vector<string>(names.begin() + i * 16, names.begin() + i * 16 + 16), ref(ioctx),
ref(listofops[i])));
if ((err = pthread_sigmask(SIG_SETMASK, &old_set, NULL)))
{
throw std::system_error(err, std::system_category(), "Failed to restore thread sigmask");
}
}
for (auto &th : threads)
{
th.join();
}
for (const auto &res : listofops)
{
all_ops.insert(all_ops.end(), res.begin(), res.end());
}
}
else
{ {
sigset_t new_set; _do_bench(settings, names, ioctx, all_ops);
sigset_t old_set;
sigfillset(&new_set);
int err;
if ((err = pthread_sigmask(SIG_SETMASK, &new_set, &old_set)))
{
throw std::system_error(err, std::system_category(), "Failed to set thread sigmask");
}
threads.push_back(thread(_do_bench, ref(settings), vector<string>(names.begin()+i*16, names.begin()+i*16+16), ref(ioctx), ref(listofops[i])));
if ((err = pthread_sigmask(SIG_SETMASK, &old_set, NULL)))
{
throw std::system_error(err, std::system_category(), "Failed to restore thread sigmask");
}
} }
print_breakdown(all_ops, settings->threads);
for (auto &th : threads)
{
th.join();
}
for (const auto &res : listofops)
{
all_ops.insert(all_ops.end(), res.begin(), res.end());
}
}
else
{
_do_bench(settings, names, ioctx, all_ops);
}
print_breakdown(all_ops, settings->threads);
} }
static void _main(int argc, const char *argv[]) static void _main(int argc, const char *argv[])
{ {
const unique_ptr<bench_settings> settings(new bench_settings); const unique_ptr<bench_settings> settings(new bench_settings);
// Default settings // Default settings
settings->secs = 10; settings->secs = 10;
settings->threads = 1; settings->threads = 1;
settings->block_size = 4096; settings->block_size = 4096;
settings->object_size = 4096 * 1024; settings->object_size = 4096 * 1024;
int ai = 1; int ai = 1;
while (ai < argc) while (ai < argc)
{
if (argv[ai][0] == '-')
{ {
if (!strcmp(argv[ai], "-d")) if (argv[ai][0] == '-')
{
// duration
++ai;
if (ai >= argc || sscanf(argv[ai], "%i", &settings->secs) != 1 ||
settings->secs < 1)
throw "Wrong duration";
}
else if (!strcmp(argv[ai], "-t"))
{
// threads
++ai;
if (ai >= argc || sscanf(argv[ai], "%i", &settings->threads) != 1 ||
settings->threads < 1)
throw "Wrong thread number";
}
else if (!strcmp(argv[ai], "-b"))
{
// block size
++ai;
if (ai >= argc || sscanf(argv[ai], "%i", (int*)&settings->block_size) != 1 ||
settings->block_size < 1)
throw "Wrong block size";
}
else if (!strcmp(argv[ai], "-o"))
{
// object size
++ai;
if (ai >= argc || sscanf(argv[ai], "%i", (int*)&settings->object_size) != 1 ||
settings->object_size < 1)
throw "Wrong object size";
}
}
else
{
if (settings->pool.empty())
settings->pool = argv[ai];
else if (settings->mode.empty())
settings->mode = argv[ai];
else if (settings->specific_bench_item.empty())
settings->specific_bench_item = argv[ai];
}
ai++;
}
if (settings->object_size < settings->block_size)
{
throw "Block size must not be greater than object size";
}
if (settings->pool.empty() || settings->mode.empty())
{
cerr << "Usage: " << argv[0]
<< " <poolname> <mode=host|osd> [specific item name to test]" << endl;
throw "Wrong cmdline";
}
Rados rados;
int err;
if ((err = rados.init("admin")) < 0)
{
cerr << "Failed to init: " << strerror(-err) << endl;
throw "Failed to init";
}
if ((err = rados.conf_read_file("/etc/ceph/ceph.conf")) < 0)
{
cerr << "Failed to read conf file: " << strerror(-err) << endl;
throw "Failed to read conf file";
}
if ((err = rados.conf_parse_argv(argc, argv)) < 0)
{
cerr << "Failed to parse argv: " << strerror(-err) << endl;
throw "Failed to parse argv";
}
if ((err = rados.connect()) < 0)
{
cerr << "Failed to connect: " << strerror(-err) << endl;
throw "Failed to connect";
}
// https://tracker.ceph.com/issues/24114
this_thread::sleep_for(milliseconds(100));
try
{
auto rados_utils = RadosUtils(&rados);
if (rados_utils.get_pool_size(settings->pool) != 1)
throw "It's required to have pool size 1";
map<unsigned int, map<string, string>> osd2location;
set<string> bench_items; // node1, node2 ||| osd.1, osd.2, osd.3
for (const auto &osd : rados_utils.get_osds(settings->pool))
{
auto location = rados_utils.get_osd_location(osd);
location["osd"] = std::to_string(osd);
osd2location[osd] = location;
if (location.find(settings->mode) != location.end())
{
const auto &osd_loc = location.at(settings->mode);
if (settings->specific_bench_item.empty() ||
osd_loc == settings->specific_bench_item)
{ {
bench_items.insert(osd_loc); if (!strcmp(argv[ai], "-d"))
{
// duration
++ai;
if (ai >= argc || sscanf(argv[ai], "%i", &settings->secs) != 1 || settings->secs < 1)
throw "Wrong duration";
}
else if (!strcmp(argv[ai], "-t"))
{
// threads
++ai;
if (ai >= argc || sscanf(argv[ai], "%i", &settings->threads) != 1 || settings->threads < 1)
throw "Wrong thread number";
}
else if (!strcmp(argv[ai], "-b"))
{
// block size
++ai;
if (ai >= argc || sscanf(argv[ai], "%i", (int *)&settings->block_size) != 1 || settings->block_size < 1)
throw "Wrong block size";
}
else if (!strcmp(argv[ai], "-o"))
{
// object size
++ai;
if (ai >= argc || sscanf(argv[ai], "%i", (int *)&settings->object_size) != 1 ||
settings->object_size < 1)
throw "Wrong object size";
}
} }
} else
else {
{ if (settings->pool.empty())
cerr << "CRUSH '" << settings->mode << "' location not found for " << osd << endl; settings->pool = argv[ai];
throw "CRUSH location not found"; else if (settings->mode.empty())
} settings->mode = argv[ai];
else if (settings->specific_bench_item.empty())
settings->specific_bench_item = argv[ai];
}
ai++;
} }
// benchitem -> [name1, name2] ||| i.e. "osd.2" => ["obj1", "obj2"] if (settings->object_size < settings->block_size)
map<string, vector<string>> name2location;
unsigned int cnt = 0;
// for each bench_item find thread_count*16 names
// store every name in name2location = [bench_item, names, description]
cout << "Finding object names" << endl;
const string prefix = "bench_";
while (bench_items.size())
{ {
string name = prefix + to_string(++cnt); throw "Block size must not be greater than object size";
unsigned int osd = rados_utils.get_obj_acting_primary(name, settings->pool);
const auto &location = osd2location.at(osd);
const auto &bench_item = location.at(settings->mode);
if (!bench_items.count(bench_item))
continue;
auto &names = name2location[bench_item];
if (names.size() >= (unsigned)settings->threads*16)
{
bench_items.erase(bench_item);
continue;
}
names.push_back(name);
} }
IoCtx ioctx; if (settings->pool.empty() || settings->mode.empty())
if (rados.ioctx_create(settings->pool.c_str(), ioctx) < 0)
throw "Failed to create ioctx";
for (const auto &p : name2location)
{ {
const auto &bench_item = p.first; cerr << "Usage: " << argv[0] << " <poolname> <mode=host|osd> [specific item name to test]" << endl;
const auto &obj_names = p.second; throw "Wrong cmdline";
cout << "Benchmarking " << settings->mode << " " << bench_item << endl; }
do_bench(settings, obj_names, ioctx);
Rados rados;
int err;
if ((err = rados.init("admin")) < 0)
{
cerr << "Failed to init: " << strerror(-err) << endl;
throw "Failed to init";
}
if ((err = rados.conf_read_file("/etc/ceph/ceph.conf")) < 0)
{
cerr << "Failed to read conf file: " << strerror(-err) << endl;
throw "Failed to read conf file";
}
if ((err = rados.conf_parse_argv(argc, argv)) < 0)
{
cerr << "Failed to parse argv: " << strerror(-err) << endl;
throw "Failed to parse argv";
}
if ((err = rados.connect()) < 0)
{
cerr << "Failed to connect: " << strerror(-err) << endl;
throw "Failed to connect";
}
// https://tracker.ceph.com/issues/24114
this_thread::sleep_for(milliseconds(100));
try
{
auto rados_utils = RadosUtils(&rados);
if (rados_utils.get_pool_size(settings->pool) != 1)
throw "It's required to have pool size 1";
map<unsigned int, map<string, string>> osd2location;
set<string> bench_items; // node1, node2 ||| osd.1, osd.2, osd.3
for (const auto &osd : rados_utils.get_osds(settings->pool))
{
auto location = rados_utils.get_osd_location(osd);
location["osd"] = std::to_string(osd);
osd2location[osd] = location;
if (location.find(settings->mode) != location.end())
{
const auto &osd_loc = location.at(settings->mode);
if (settings->specific_bench_item.empty() || osd_loc == settings->specific_bench_item)
{
bench_items.insert(osd_loc);
}
}
else
{
cerr << "CRUSH '" << settings->mode << "' location not found for " << osd << endl;
throw "CRUSH location not found";
}
}
// benchitem -> [name1, name2] ||| i.e. "osd.2" => ["obj1", "obj2"]
map<string, vector<string>> name2location;
unsigned int cnt = 0;
// for each bench_item find thread_count*16 names
// store every name in name2location = [bench_item, names, description]
cout << "Finding object names" << endl;
const string prefix = "bench_";
while (bench_items.size())
{
string name = prefix + to_string(++cnt);
unsigned int osd = rados_utils.get_obj_acting_primary(name, settings->pool);
const auto &location = osd2location.at(osd);
const auto &bench_item = location.at(settings->mode);
if (!bench_items.count(bench_item))
continue;
auto &names = name2location[bench_item];
if (names.size() >= (unsigned)settings->threads * 16)
{
bench_items.erase(bench_item);
continue;
}
names.push_back(name);
}
IoCtx ioctx;
if (rados.ioctx_create(settings->pool.c_str(), ioctx) < 0)
throw "Failed to create ioctx";
for (const auto &p : name2location)
{
const auto &bench_item = p.first;
const auto &obj_names = p.second;
cout << "Benchmarking " << settings->mode << " " << bench_item << endl;
do_bench(settings, obj_names, ioctx);
}
}
catch (...)
{
rados.watch_flush();
throw;
} }
}
catch (...)
{
rados.watch_flush(); rados.watch_flush();
throw;
}
rados.watch_flush();
//rados_ioctx_destroy(io); // rados_ioctx_destroy(io);
//rados_shutdown(cluster); // rados_shutdown(cluster);
} }
int main(int argc, const char *argv[]) int main(int argc, const char *argv[])
{ {
try try
{ {
setup_signal_handlers(); setup_signal_handlers();
_main(argc, argv); _main(argc, argv);
} }
catch (const AbortException &msg) catch (const AbortException &msg)
{ {
cerr << "Test aborted" << endl; cerr << "Test aborted" << endl;
return 1; return 1;
} }
catch (const char *msg) catch (const char *msg)
{ {
cerr << "Unhandled exception: " << msg << endl; cerr << "Unhandled exception: " << msg << endl;
return 2; return 2;
} }
cout << "Exiting successfully." << endl; cout << "Exiting successfully." << endl;
return 0; return 0;
} }

25
mysignals.cpp
View File

@ -1,16 +1,21 @@
#include <csignal>
#include "mysignals.h" #include "mysignals.h"
#include <csignal>
static volatile std::sig_atomic_t gSignalStatus; static volatile std::sig_atomic_t gSignalStatus;
static void signal_handler(int signal) { gSignalStatus = signal; } static void signal_handler(int signal)
{
void setup_signal_handlers() { gSignalStatus = signal;
std::signal(SIGINT, signal_handler);
std::signal(SIGTERM, signal_handler);
} }
void abort_if_signalled() { void setup_signal_handlers()
if (gSignalStatus) {
throw AbortException(); std::signal(SIGINT, signal_handler);
std::signal(SIGTERM, signal_handler);
}
void abort_if_signalled()
{
if (gSignalStatus)
throw AbortException();
} }

4
mysignals.h
View File

@ -5,7 +5,9 @@
void setup_signal_handlers(); void setup_signal_handlers();
class AbortException : public std::exception {}; class AbortException : public std::exception
{
};
void abort_if_signalled(); void abort_if_signalled();
#endif #endif

134
radosutil.cpp
View File

@ -1,9 +1,9 @@
#include <string> #include "radosutil.h"
#include <json/json.h> #include <json/json.h>
#include <librados.hpp>
#include "radosutil.h" #include <librados.hpp>
#include <string>
using namespace std; using namespace std;
using namespace librados; using namespace librados;
@ -12,93 +12,99 @@ RadosUtils::RadosUtils(Rados *rados_)
: rados(rados_), /* */ : rados(rados_), /* */
json_reader(new Json::Reader(Json::Features::strictMode())), /* */ json_reader(new Json::Reader(Json::Features::strictMode())), /* */
json_writer(new Json::FastWriter()) /* */ json_writer(new Json::FastWriter()) /* */
{} {
}
// RadosUtils::~RadosUtils() {} // RadosUtils::~RadosUtils() {}
unsigned int RadosUtils::get_obj_acting_primary(const string &name, unsigned int RadosUtils::get_obj_acting_primary(const string &name, const string &pool)
const string &pool) { {
Json::Value cmd(Json::objectValue);
cmd["prefix"] = "osd map";
cmd["object"] = name;
cmd["pool"] = pool;
Json::Value cmd(Json::objectValue); auto &&location = do_mon_command(cmd);
cmd["prefix"] = "osd map";
cmd["object"] = name;
cmd["pool"] = pool;
auto &&location = do_mon_command(cmd); const auto &acting_primary = location["acting_primary"];
if (!acting_primary.isNumeric())
throw "Failed to get acting_primary";
const auto &acting_primary = location["acting_primary"]; return acting_primary.asUInt();
if (!acting_primary.isNumeric())
throw "Failed to get acting_primary";
return acting_primary.asUInt();
} }
map<string, string> RadosUtils::get_osd_location(unsigned int osd) { map<string, string> RadosUtils::get_osd_location(unsigned int osd)
Json::Value cmd(Json::objectValue); {
cmd["prefix"] = "osd find"; Json::Value cmd(Json::objectValue);
cmd["id"] = osd; cmd["prefix"] = "osd find";
cmd["id"] = osd;
auto &&location = do_mon_command(cmd); auto &&location = do_mon_command(cmd);
const auto &crush = location["crush_location"]; const auto &crush = location["crush_location"];
map<string, string> result; map<string, string> result;
for (auto &&it = crush.begin(); it != crush.end(); ++it) { for (auto &&it = crush.begin(); it != crush.end(); ++it)
result[it.name()] = it->asString(); {
} result[it.name()] = it->asString();
}
result["osd"] = "osd." + to_string(osd); result["osd"] = "osd." + to_string(osd);
return result; return result;
} }
set<unsigned int> RadosUtils::get_osds(const string &pool) { set<unsigned int> RadosUtils::get_osds(const string &pool)
Json::Value cmd(Json::objectValue); {
cmd["prefix"] = "pg ls-by-pool"; Json::Value cmd(Json::objectValue);
cmd["poolstr"] = pool; cmd["prefix"] = "pg ls-by-pool";
cmd["poolstr"] = pool;
auto pgs = do_mon_command(cmd); auto pgs = do_mon_command(cmd);
set<unsigned int> osds; set<unsigned int> osds;
if (pgs.isMember("pg_stats")) { if (pgs.isMember("pg_stats"))
pgs = pgs["pg_stats"]; {
} pgs = pgs["pg_stats"];
for (const auto &pg : pgs) { }
const auto &primary = pg["acting_primary"]; for (const auto &pg : pgs)
if (!primary.isNumeric()) {
throw "Failed to get acting_primary"; const auto &primary = pg["acting_primary"];
osds.insert(primary.asUInt()); if (!primary.isNumeric())
} throw "Failed to get acting_primary";
osds.insert(primary.asUInt());
}
return osds; return osds;
} }
unsigned int RadosUtils::get_pool_size(const string &pool) { unsigned int RadosUtils::get_pool_size(const string &pool)
Json::Value cmd(Json::objectValue); {
cmd["prefix"] = "osd pool get"; Json::Value cmd(Json::objectValue);
cmd["pool"] = pool; cmd["prefix"] = "osd pool get";
cmd["var"] = "size"; cmd["pool"] = pool;
cmd["var"] = "size";
const auto &&v = do_mon_command(cmd); const auto &&v = do_mon_command(cmd);
return v["size"].asUInt(); return v["size"].asUInt();
} }
Json::Value RadosUtils::do_mon_command(Json::Value &cmd) { Json::Value RadosUtils::do_mon_command(Json::Value &cmd)
int err; {
bufferlist outbl; int err;
string outs; bufferlist outbl;
cmd["format"] = "json"; string outs;
bufferlist inbl; cmd["format"] = "json";
bufferlist inbl;
if ((err = rados->mon_command(json_writer->write(cmd), inbl, &outbl, &outs)) < if ((err = rados->mon_command(json_writer->write(cmd), inbl, &outbl, &outs)) < 0)
0) throw MyRadosException(err, outs);
throw MyRadosException(err, outs);
Json::Value root; Json::Value root;
if (!json_reader->parse(outbl.to_str(), root)) if (!json_reader->parse(outbl.to_str(), root))
throw "JSON parse error"; throw "JSON parse error";
return root; return root;
} }

50
radosutil.h
View File

@ -1,39 +1,45 @@
#include <json/json.h>
#include <exception> #include <exception>
#include <map> #include <map>
#include <memory> #include <memory>
#include <set> #include <set>
#include <string> #include <string>
#include <json/json.h>
#include "librados_fwd.hpp" #include "librados_fwd.hpp"
namespace Json { namespace Json
{
class Reader; class Reader;
class FastWriter; class FastWriter;
class Value; class Value;
} // namespace Json } // namespace Json
class RadosUtils { class RadosUtils
public: {
explicit RadosUtils(librados::Rados *rados_); public:
unsigned int get_obj_acting_primary(const std::string &name, explicit RadosUtils(librados::Rados *rados_);
const std::string &pool); unsigned int get_obj_acting_primary(const std::string &name, const std::string &pool);
std::map<std::string, std::string> get_osd_location(unsigned int osd); std::map<std::string, std::string> get_osd_location(unsigned int osd);
std::set<unsigned int> get_osds(const std::string &pool); std::set<unsigned int> get_osds(const std::string &pool);
unsigned int get_pool_size(const std::string &pool); unsigned int get_pool_size(const std::string &pool);
private: private:
Json::Value do_mon_command(Json::Value &cmd); Json::Value do_mon_command(Json::Value &cmd);
librados::Rados *rados; librados::Rados *rados;
std::unique_ptr<Json::Reader> json_reader; std::unique_ptr<Json::Reader> json_reader;
std::unique_ptr<Json::FastWriter> json_writer; std::unique_ptr<Json::FastWriter> json_writer;
}; };
class MyRadosException : public std::exception { class MyRadosException : public std::exception
public: {
MyRadosException(int err, const std::string &msg) public:
: descr("Rados err " + std::to_string(err) + ": " + msg){}; MyRadosException(int err, const std::string &msg) : descr("Rados err " + std::to_string(err) + ": " + msg){};
const char *what() const throw() { return descr.c_str(); } const char *what() const throw()
{
return descr.c_str();
}
private: private:
std::string descr; std::string descr;
}; };