Begin to implement CLI: implement listing, add help, add create stub

nbd-vmsplice
Vitaliy Filippov 2021-11-05 02:33:51 +03:00
parent ffb06536ff
commit 8e445ddc9a
8 changed files with 405 additions and 18 deletions

2
json11

@ -1 +1 @@
Subproject commit 97f06cb20c1e136fd37d58fb40f57dd8f8a3a4a7
Subproject commit 3a5b4477bc011e3224a3b0f9e2883acc78eec14c

View File

@ -153,7 +153,7 @@ target_link_libraries(vitastor-nbd
# vitastor-cli
add_executable(vitastor-cli
cli.cpp cli_flatten.cpp cli_merge.cpp cli_rm.cpp cli_snap_rm.cpp
cli.cpp cli_ls.cpp cli_flatten.cpp cli_merge.cpp cli_rm.cpp cli_snap_rm.cpp
)
target_link_libraries(vitastor-cli
vitastor_client

View File

@ -28,10 +28,24 @@ json11::Json::object cli_tool_t::parse_args(int narg, const char *args[])
{
help();
}
else if (args[i][0] == '-' && args[i][1] == 'l')
{
cfg["long"] = "1";
}
else if (args[i][0] == '-' && args[i][1] == 'n')
{
cfg["count"] = args[++i];
}
else if (args[i][0] == '-' && args[i][1] == 'i')
{
cfg["interactive"] = "1";
}
else if (args[i][0] == '-' && args[i][1] == '-')
{
const char *opt = args[i]+2;
cfg[opt] = !strcmp(opt, "json") || !strcmp(opt, "wait-list") || i == narg-1 ? "1" : args[++i];
cfg[opt] = i == narg-1 || !strcmp(opt, "json") || !strcmp(opt, "wait-list") ||
!strcmp(opt, "long") || !strcmp(opt, "writers-stopped") && strcmp("1", args[i+1]) != 0
? "1" : args[++i];
}
else
{
@ -54,35 +68,54 @@ void cli_tool_t::help()
{
printf(
"Vitastor command-line tool\n"
"(c) Vitaliy Filippov, 2019+ (VNPL-1.1)\n\n"
"(c) Vitaliy Filippov, 2019+ (VNPL-1.1)\n"
"\n"
"USAGE:\n"
"%s ls [-l]\n"
" List existing images. Also report provisioned and allocated size if -l is specified.\n"
"\n"
"%s create --size <size> [--parent <parent_name>[@<snapshot>]] <name>\n"
" Create an image. You may use K/M/G/T suffixes for <size>. If --parent is specified,\n"
" a copy-on-write image clone is created. Parent must be a snapshot (readonly image).\n"
"\n"
"%s snap-create <name>@<snapshot>\n"
" Create a snapshot of image <name>. May be used live if only a single writer is active.\n"
"\n"
"%s set <name> [--size <size>] [--readonly | --readwrite]\n"
" Resize image or change its readonly status. Images with children can't be made read-write.\n"
"\n"
"%s top [-n <MAX_COUNT>] [-i]\n"
" Disable image list sorted by I/O load, interactive if -i specified.\n"
"\n"
"%s rm [OPTIONS] <from> [<to>] [--writers-stopped]\n"
" Remove <from> or all layers between <from> and <to> (<to> must be a child of <from>),\n"
" rebasing all their children accordingly. --writers-stopped allows merging to be a bit\n"
" more effective in case of a single 'slim' read-write child and 'fat' removed parent:\n"
" the child is merged into parent in that case and parent is renamed to child.\n"
" In other cases parent layers are always merged into children.\n"
"\n"
"%s flatten [OPTIONS] <layer>\n"
" Flatten a layer, i.e. merge data and detach it from parents.\n"
"\n"
"%s rm-data [OPTIONS] --pool <pool> --inode <inode> [--wait-list]\n"
" Remove inode data without changing metadata.\n"
" --wait-list means first retrieve objects listings and then remove it.\n"
" --wait-list requires more memory, but allows to show correct stats.\n"
" --wait-list requires more memory, but allows to show correct removal progress.\n"
"\n"
"%s merge-data [OPTIONS] <from> <to> [--target <target>]\n"
" Merge layer data without changing metadata. Merge <from>..<to> to <target>.\n"
" <to> must be a child of <from> and <target> may be one of the layers between\n"
" <from> and <to>, including <from> and <to>.\n"
"\n"
"%s flatten [OPTIONS] <layer>\n"
" Flatten a layer, i.e. merge data and detach it from parents\n"
"\n"
"%s rm [OPTIONS] <from> [<to>] [--writers-stopped 1]\n"
" Remove <from> or all layers between <from> and <to> (<to> must be a child of <from>),\n"
" rebasing all their children accordingly. One of deleted parents may be renamed to one\n"
" of children \"to be rebased\", but only if that child itself is readonly or if\n"
" --writers-stopped 1 is specified\n"
"\n"
"OPTIONS (global):\n"
" --etcd_address <etcd_address>\n"
" --iodepth N Send N operations in parallel to each OSD when possible (default 32)\n"
" --parallel_osds M Work with M osds in parallel when possible (default 4)\n"
" --progress 1|0 Report progress (default 1)\n"
" --cas 1|0 Use online CAS writes when possible (default auto)\n"
" --json JSON output\n"
,
exe_name, exe_name, exe_name, exe_name
exe_name, exe_name, exe_name, exe_name, exe_name, exe_name, exe_name, exe_name, exe_name
);
exit(0);
}
@ -176,6 +209,11 @@ void cli_tool_t::run(json11::Json cfg)
fprintf(stderr, "command is missing\n");
exit(1);
}
else if (cmd[0] == "ls")
{
// List images
action_cb = start_ls(cfg);
}
else if (cmd[0] == "rm-data")
{
// Delete inode data
@ -201,6 +239,7 @@ void cli_tool_t::run(json11::Json cfg)
fprintf(stderr, "unknown command: %s\n", cmd[0].string_value().c_str());
exit(1);
}
json_output = cfg["json"].bool_value();
iodepth = cfg["iodepth"].uint64_value();
if (!iodepth)
iodepth = 32;
@ -236,7 +275,8 @@ void cli_tool_t::run(json11::Json cfg)
while (action_cb != NULL)
{
ringloop->loop();
ringloop->wait();
if (action_cb != NULL)
ringloop->wait();
}
}

View File

@ -25,6 +25,7 @@ public:
uint64_t iodepth = 0, parallel_osds = 0;
bool progress = true;
bool list_first = false;
bool json_output = false;
int log_level = 0;
int mode = 0;
@ -49,6 +50,7 @@ public:
friend struct snap_flattener_t;
friend struct snap_remover_t;
std::function<bool(void)> start_ls(json11::Json cfg);
std::function<bool(void)> start_rm(json11::Json);
std::function<bool(void)> start_merge(json11::Json);
std::function<bool(void)> start_flatten(json11::Json);

50
src/cli_create.cpp Normal file
View File

@ -0,0 +1,50 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#include "cli.h"
#include "cluster_client.h"
#include "base64.h"
// Create an image, snapshot or clone
//
// Snapshot creation does a etcd transaction which:
// - Changes the name of old inode to the name of the snapshot (say, testimg -> testimg@0)
// - Sets the readonly flag for the old inode
// - Creates a new inode with the same name pointing to the old inode as parent
// - Adjusts /index/image/*
//
// The same algorithm can be easily implemented in any other language or even via etcdctl,
// however we have it here for completeness
struct image_creator_t
{
cli_tool_t *parent;
int state = 0;
bool is_done()
{
return true;
}
void loop()
{
return;
}
};
std::function<bool(void)> cli_tool_t::start_create(json11::Json cfg)
{
json11::Json::array cmd = cfg["command"].array_items();
auto image_creator = new image_creator_t();
image_creator->parent = this;
return [image_creator]()
{
image_creator->loop();
if (image_creator->is_done())
{
delete image_creator;
return true;
}
return false;
};
}

295
src/cli_ls.cpp Normal file
View File

@ -0,0 +1,295 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#include "cli.h"
#include "cluster_client.h"
#include "base64.h"
std::string print_table(json11::Json items, json11::Json header);
std::string format_size(uint64_t size);
// List existing images
//
// Again, you can just look into etcd, but this console tool incapsulates it
struct image_lister_t
{
cli_tool_t *parent;
int state = 0;
bool detailed = false;
std::map<inode_t, uint64_t> used_sizes;
json11::Json space_info;
bool is_done()
{
return state == 100;
}
json11::Json::array get_list()
{
json11::Json::array list;
for (auto & ic: parent->cli->st_cli.inode_config)
{
auto item = json11::Json::object {
{ "name", ic.second.name },
{ "size", ic.second.size },
{ "used_size", used_sizes[ic.second.num] },
{ "readonly", ic.second.readonly },
{ "pool_id", (uint64_t)INODE_POOL(ic.second.num) },
{ "inode_num", INODE_NO_POOL(ic.second.num) },
};
if (ic.second.parent_id)
{
auto p_it = parent->cli->st_cli.inode_config.find(ic.second.parent_id);
item["parent_name"] = p_it != parent->cli->st_cli.inode_config.end()
? p_it->second.name : "";
item["parent_pool_id"] = (uint64_t)INODE_POOL(ic.second.parent_id);
item["parent_inode_num"] = INODE_NO_POOL(ic.second.parent_id);
}
if (!parent->json_output)
{
item["used_size_fmt"] = format_size(used_sizes[ic.second.num]);
item["size_fmt"] = format_size(ic.second.size);
item["ro"] = ic.second.readonly ? "RO" : "-";
}
list.push_back(item);
}
return list;
}
void loop()
{
if (state == 1)
goto resume_1;
if (detailed)
{
// Space statistics
// inode/stats/<pool>/<inode>::raw_used divided by pool/stats/<pool>::pg_real_size
// multiplied by 1 or number of data drives
parent->waiting++;
parent->cli->st_cli.etcd_txn(json11::Json::object {
{ "success", json11::Json::array {
json11::Json::object {
{ "request_range", json11::Json::object {
{ "key", base64_encode(
parent->cli->st_cli.etcd_prefix+"/pool/stats/"
) },
{ "range_end", base64_encode(
parent->cli->st_cli.etcd_prefix+"/pool/stats0"
) },
} },
},
json11::Json::object {
{ "request_range", json11::Json::object {
{ "key", base64_encode(
parent->cli->st_cli.etcd_prefix+"/inode/stats/"
) },
{ "range_end", base64_encode(
parent->cli->st_cli.etcd_prefix+"/inode/stats0"
) },
} },
},
} },
}, ETCD_SLOW_TIMEOUT, [this](std::string err, json11::Json res)
{
parent->waiting--;
if (err != "")
{
fprintf(stderr, "Error reading from etcd: %s\n", err.c_str());
exit(1);
}
space_info = res;
});
state = 1;
resume_1:
if (parent->waiting > 0)
return;
std::map<pool_id_t, uint64_t> pool_pg_real_size;
for (auto & kv_item: space_info["responses"][0]["response_range"]["kvs"].array_items())
{
auto kv = parent->cli->st_cli.parse_etcd_kv(kv_item);
// pool ID
pool_id_t pool_id;
char null_byte = 0;
sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(), "/pool/stats/%u%c", &pool_id, &null_byte);
if (!pool_id || pool_id >= POOL_ID_MAX || null_byte != 0)
{
fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
continue;
}
// pg_real_size
pool_pg_real_size[pool_id] = kv.value["pg_real_size"].uint64_value();
}
for (auto & kv_item: space_info["responses"][1]["response_range"]["kvs"].array_items())
{
auto kv = parent->cli->st_cli.parse_etcd_kv(kv_item);
// pool ID & inode number
pool_id_t pool_id;
inode_t inode_num;
char null_byte = 0;
sscanf(kv.key.substr(parent->cli->st_cli.etcd_prefix.length()).c_str(), "/inode/stats/%u/%lu%c", &pool_id, &inode_num, &null_byte);
if (!pool_id || pool_id >= POOL_ID_MAX || INODE_POOL(inode_num) != 0 || null_byte != 0)
{
fprintf(stderr, "Invalid key in etcd: %s\n", kv.key.c_str());
continue;
}
// save stats
auto & pool_cfg = parent->cli->st_cli.pool_config.at(pool_id);
used_sizes[INODE_WITH_POOL(pool_id, inode_num)] = kv.value["raw_used"].uint64_value() / pool_pg_real_size[pool_id]
* (pool_cfg.scheme == POOL_SCHEME_REPLICATED ? 1 : pool_cfg.pg_size-pool_cfg.parity_chunks);
}
}
json11::Json::array list = get_list();
if (parent->json_output)
{
// JSON output
printf("%s\n", json11::Json(list).dump().c_str());
state = 100;
return;
}
// Table output: name, size_fmt, [used_size_fmt], ro, parent_name
json11::Json::array cols = json11::Json::array{
json11::Json::object{
{ "key", "name" },
{ "title", "NAME" },
},
json11::Json::object{
{ "key", "size_fmt" },
{ "title", "SIZE" },
{ "right", true },
},
};
if (detailed)
{
cols.push_back(json11::Json::object{
{ "key", "used_size_fmt" },
{ "title", "USED" },
{ "right", true },
});
}
cols.push_back(json11::Json::object{
{ "key", "ro" },
{ "title", "FLAGS" },
{ "right", true },
});
cols.push_back(json11::Json::object{
{ "key", "parent_name" },
{ "title", "PARENT" },
});
printf("%s", print_table(list, cols).c_str());
state = 100;
}
};
std::string print_table(json11::Json items, json11::Json header)
{
std::vector<int> sizes;
for (int i = 0; i < header.array_items().size(); i++)
{
sizes.push_back(header[i]["title"].string_value().length());
}
for (auto & item: items.array_items())
{
for (int i = 0; i < header.array_items().size(); i++)
{
int l = item[header[i]["key"].string_value()].string_value().length();
sizes[i] = sizes[i] < l ? l : sizes[i];
}
}
std::string str = "";
for (int i = 0; i < header.array_items().size(); i++)
{
if (i > 0)
{
// Separator
str += " ";
}
int pad = sizes[i]-header[i]["title"].string_value().length();
if (header[i]["right"].bool_value())
{
// Align right
for (int j = 0; j < pad; j++)
str += ' ';
str += header[i]["title"].string_value();
}
else
{
// Align left
str += header[i]["title"].string_value();
for (int j = 0; j < pad; j++)
str += ' ';
}
}
str += "\n";
for (auto & item: items.array_items())
{
for (int i = 0; i < header.array_items().size(); i++)
{
if (i > 0)
{
// Separator
str += " ";
}
int pad = sizes[i] - item[header[i]["key"].string_value()].string_value().length();
if (header[i]["right"].bool_value())
{
// Align right
for (int j = 0; j < pad; j++)
str += ' ';
str += item[header[i]["key"].string_value()].string_value();
}
else
{
// Align left
str += item[header[i]["key"].string_value()].string_value();
for (int j = 0; j < pad; j++)
str += ' ';
}
}
str += "\n";
}
return str;
}
static uint64_t size_thresh[] = { 1024l*1024*1024*1024, 1024l*1024*1024, 1024l*1024, 1024 };
static const char *size_unit = "TGMK";
std::string format_size(uint64_t size)
{
char buf[256];
for (int i = 0; i < sizeof(size_thresh)/sizeof(size_thresh[0]); i++)
{
if (size >= size_thresh[i] || i >= sizeof(size_thresh)/sizeof(size_thresh[0])-1)
{
double value = (double)size/size_thresh[i];
int l = snprintf(buf, sizeof(buf), "%.1f", value);
assert(l < sizeof(buf)-2);
if (buf[l-1] == '0')
l -= 2;
buf[l] = ' ';
buf[l+1] = size_unit[i];
buf[l+2] = 0;
break;
}
}
return std::string(buf);
}
std::function<bool(void)> cli_tool_t::start_ls(json11::Json cfg)
{
json11::Json::array cmd = cfg["command"].array_items();
auto lister = new image_lister_t();
lister->parent = this;
lister->detailed = cfg["long"].bool_value();
return [lister]()
{
lister->loop();
if (lister->is_done())
{
delete lister;
return true;
}
return false;
};
}

View File

@ -766,7 +766,7 @@ void etcd_state_client_t::close_watch(inode_watch_t* watch)
delete watch;
}
json11::Json::object & etcd_state_client_t::serialize_inode_cfg(inode_config_t *cfg)
json11::Json::object etcd_state_client_t::serialize_inode_cfg(inode_config_t *cfg)
{
json11::Json::object new_cfg = json11::Json::object {
{ "name", cfg->name },

View File

@ -99,7 +99,7 @@ public:
std::function<void(pool_id_t, pg_num_t)> on_change_pg_history_hook;
std::function<void(osd_num_t)> on_change_osd_state_hook;
json11::Json::object & serialize_inode_cfg(inode_config_t *cfg);
json11::Json::object serialize_inode_cfg(inode_config_t *cfg);
etcd_kv_t parse_etcd_kv(const json11::Json & kv_json);
void etcd_call(std::string api, json11::Json payload, int timeout, std::function<void(std::string, json11::Json)> callback);
void etcd_txn(json11::Json txn, int timeout, std::function<void(std::string, json11::Json)> callback);