parent
3de553ecd7
commit
511a89948b
@ -1,177 +0,0 @@ |
||||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
// C-C++ proxy for the QEMU driver
|
||||
// (QEMU headers don't compile with g++)
|
||||
|
||||
#include <sys/epoll.h> |
||||
|
||||
#include "cluster_client.h" |
||||
|
||||
typedef void* AioContext; |
||||
#include "qemu_proxy.h" |
||||
|
||||
extern "C" |
||||
{ |
||||
// QEMU
|
||||
typedef void IOHandler(void *opaque); |
||||
void aio_set_fd_handler(AioContext *ctx, int fd, int is_external, IOHandler *fd_read, IOHandler *fd_write, void *poll_fn, void *opaque); |
||||
} |
||||
|
||||
struct QemuProxyData |
||||
{ |
||||
int fd; |
||||
std::function<void(int, int)> callback; |
||||
}; |
||||
|
||||
class QemuProxy |
||||
{ |
||||
std::map<int, QemuProxyData> handlers; |
||||
|
||||
public: |
||||
|
||||
timerfd_manager_t *tfd; |
||||
cluster_client_t *cli; |
||||
AioContext *ctx; |
||||
|
||||
QemuProxy(AioContext *ctx, const char *config_path, const char *etcd_host, const char *etcd_prefix, |
||||
const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu) |
||||
{ |
||||
this->ctx = ctx; |
||||
json11::Json::object cfg; |
||||
if (config_path) |
||||
cfg["config_path"] = std::string(config_path); |
||||
if (etcd_host) |
||||
cfg["etcd_address"] = std::string(etcd_host); |
||||
if (etcd_prefix) |
||||
cfg["etcd_prefix"] = std::string(etcd_prefix); |
||||
if (rdma_device) |
||||
cfg["rdma_device"] = std::string(rdma_device); |
||||
if (rdma_port_num) |
||||
cfg["rdma_port_num"] = rdma_port_num; |
||||
if (rdma_gid_index) |
||||
cfg["rdma_gid_index"] = rdma_gid_index; |
||||
if (rdma_mtu) |
||||
cfg["rdma_mtu"] = rdma_mtu; |
||||
json11::Json cfg_json(cfg); |
||||
tfd = new timerfd_manager_t([this](int fd, bool wr, std::function<void(int, int)> callback) { set_fd_handler(fd, wr, callback); }); |
||||
cli = new cluster_client_t(NULL, tfd, cfg_json); |
||||
} |
||||
|
||||
~QemuProxy() |
||||
{ |
||||
delete cli; |
||||
delete tfd; |
||||
} |
||||
|
||||
void set_fd_handler(int fd, bool wr, std::function<void(int, int)> callback) |
||||
{ |
||||
if (callback != NULL) |
||||
{ |
||||
handlers[fd] = { .fd = fd, .callback = callback }; |
||||
aio_set_fd_handler(ctx, fd, false, &QemuProxy::read_handler, wr ? &QemuProxy::write_handler : NULL, NULL, &handlers[fd]); |
||||
} |
||||
else |
||||
{ |
||||
handlers.erase(fd); |
||||
aio_set_fd_handler(ctx, fd, false, NULL, NULL, NULL, NULL); |
||||
} |
||||
} |
||||
|
||||
static void read_handler(void *opaque) |
||||
{ |
||||
QemuProxyData *data = (QemuProxyData *)opaque; |
||||
data->callback(data->fd, EPOLLIN); |
||||
} |
||||
|
||||
static void write_handler(void *opaque) |
||||
{ |
||||
QemuProxyData *data = (QemuProxyData *)opaque; |
||||
data->callback(data->fd, EPOLLOUT); |
||||
} |
||||
}; |
||||
|
||||
extern "C" { |
||||
|
||||
void* vitastor_proxy_create(AioContext *ctx, const char *config_path, const char *etcd_host, const char *etcd_prefix, |
||||
const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu) |
||||
{ |
||||
QemuProxy *p = new QemuProxy(ctx, config_path, etcd_host, etcd_prefix, rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu); |
||||
return p; |
||||
} |
||||
|
||||
void vitastor_proxy_destroy(void *client) |
||||
{ |
||||
QemuProxy *p = (QemuProxy*)client; |
||||
delete p; |
||||
} |
||||
|
||||
void vitastor_proxy_rw(int write, void *client, uint64_t inode, uint64_t offset, uint64_t len, |
||||
iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque) |
||||
{ |
||||
QemuProxy *p = (QemuProxy*)client; |
||||
cluster_op_t *op = new cluster_op_t; |
||||
op->opcode = write ? OSD_OP_WRITE : OSD_OP_READ; |
||||
op->inode = inode; |
||||
op->offset = offset; |
||||
op->len = len; |
||||
for (int i = 0; i < iovcnt; i++) |
||||
{ |
||||
op->iov.push_back(iov[i].iov_base, iov[i].iov_len); |
||||
} |
||||
op->callback = [cb, opaque](cluster_op_t *op) |
||||
{ |
||||
cb(op->retval, opaque); |
||||
delete op; |
||||
}; |
||||
p->cli->execute(op); |
||||
} |
||||
|
||||
void vitastor_proxy_sync(void *client, VitastorIOHandler cb, void *opaque) |
||||
{ |
||||
QemuProxy *p = (QemuProxy*)client; |
||||
cluster_op_t *op = new cluster_op_t; |
||||
op->opcode = OSD_OP_SYNC; |
||||
op->callback = [cb, opaque](cluster_op_t *op) |
||||
{ |
||||
cb(op->retval, opaque); |
||||
delete op; |
||||
}; |
||||
p->cli->execute(op); |
||||
} |
||||
|
||||
void vitastor_proxy_watch_metadata(void *client, char *image, VitastorIOHandler cb, void *opaque) |
||||
{ |
||||
QemuProxy *p = (QemuProxy*)client; |
||||
p->cli->on_ready([=]() |
||||
{ |
||||
auto watch = p->cli->st_cli.watch_inode(std::string(image)); |
||||
cb((long)watch, opaque); |
||||
}); |
||||
} |
||||
|
||||
void vitastor_proxy_close_watch(void *client, void *watch) |
||||
{ |
||||
QemuProxy *p = (QemuProxy*)client; |
||||
p->cli->st_cli.close_watch((inode_watch_t*)watch); |
||||
} |
||||
|
||||
uint64_t vitastor_proxy_get_size(void *watch_ptr) |
||||
{ |
||||
inode_watch_t *watch = (inode_watch_t*)watch_ptr; |
||||
return watch->cfg.size; |
||||
} |
||||
|
||||
uint64_t vitastor_proxy_get_inode_num(void *watch_ptr) |
||||
{ |
||||
inode_watch_t *watch = (inode_watch_t*)watch_ptr; |
||||
return watch->cfg.num; |
||||
} |
||||
|
||||
int vitastor_proxy_get_readonly(void *watch_ptr) |
||||
{ |
||||
inode_watch_t *watch = (inode_watch_t*)watch_ptr; |
||||
return watch->cfg.readonly; |
||||
} |
||||
|
||||
} |
||||
@ -1,35 +0,0 @@ |
||||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
#ifndef VITASTOR_QEMU_PROXY_H |
||||
#define VITASTOR_QEMU_PROXY_H |
||||
|
||||
#ifndef POOL_ID_BITS |
||||
#define POOL_ID_BITS 16 |
||||
#endif |
||||
#include <stdint.h> |
||||
#include <sys/uio.h> |
||||
|
||||
#ifdef __cplusplus |
||||
extern "C" { |
||||
#endif |
||||
|
||||
// Our exports
|
||||
typedef void VitastorIOHandler(long retval, void *opaque); |
||||
void* vitastor_proxy_create(AioContext *ctx, const char *config_path, const char *etcd_host, const char *etcd_prefix, |
||||
const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu); |
||||
void vitastor_proxy_destroy(void *client); |
||||
void vitastor_proxy_rw(int write, void *client, uint64_t inode, uint64_t offset, uint64_t len, |
||||
struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque); |
||||
void vitastor_proxy_sync(void *client, VitastorIOHandler cb, void *opaque); |
||||
void vitastor_proxy_watch_metadata(void *client, char *image, VitastorIOHandler cb, void *opaque); |
||||
void vitastor_proxy_close_watch(void *client, void *watch); |
||||
uint64_t vitastor_proxy_get_size(void *watch); |
||||
uint64_t vitastor_proxy_get_inode_num(void *watch); |
||||
int vitastor_proxy_get_readonly(void *watch); |
||||
|
||||
#ifdef __cplusplus |
||||
} |
||||
#endif |
||||
|
||||
#endif |
||||
@ -0,0 +1,245 @@ |
||||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
// Simplified C client library for QEMU, fio and other external drivers
|
||||
// Also acts as a C-C++ proxy for the QEMU driver (QEMU headers don't compile with g++)
|
||||
|
||||
#include <sys/epoll.h> |
||||
|
||||
#include "ringloop.h" |
||||
#include "epoll_manager.h" |
||||
#include "cluster_client.h" |
||||
|
||||
#include "vitastor_c.h" |
||||
|
||||
struct vitastor_qemu_fd_t |
||||
{ |
||||
int fd; |
||||
std::function<void(int, int)> callback; |
||||
}; |
||||
|
||||
struct vitastor_c |
||||
{ |
||||
std::map<int, vitastor_qemu_fd_t> handlers; |
||||
ring_loop_t *ringloop = NULL; |
||||
epoll_manager_t *epmgr = NULL; |
||||
timerfd_manager_t *tfd = NULL; |
||||
cluster_client_t *cli = NULL; |
||||
|
||||
QEMUSetFDHandler *aio_set_fd_handler = NULL; |
||||
void *aio_ctx = NULL; |
||||
}; |
||||
|
||||
extern "C" { |
||||
|
||||
static json11::Json vitastor_c_common_config(const char *config_path, const char *etcd_host, const char *etcd_prefix, |
||||
int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level) |
||||
{ |
||||
json11::Json::object cfg; |
||||
if (config_path) |
||||
cfg["config_path"] = std::string(config_path); |
||||
if (etcd_host) |
||||
cfg["etcd_address"] = std::string(etcd_host); |
||||
if (etcd_prefix) |
||||
cfg["etcd_prefix"] = std::string(etcd_prefix); |
||||
// -1 means unspecified
|
||||
if (use_rdma >= 0) |
||||
cfg["use_rdma"] = use_rdma > 0; |
||||
if (rdma_device) |
||||
cfg["rdma_device"] = std::string(rdma_device); |
||||
if (rdma_port_num) |
||||
cfg["rdma_port_num"] = rdma_port_num; |
||||
if (rdma_gid_index) |
||||
cfg["rdma_gid_index"] = rdma_gid_index; |
||||
if (rdma_mtu) |
||||
cfg["rdma_mtu"] = rdma_mtu; |
||||
if (log_level) |
||||
cfg["log_level"] = log_level; |
||||
return json11::Json(cfg); |
||||
} |
||||
|
||||
static void vitastor_c_read_handler(void *opaque) |
||||
{ |
||||
vitastor_qemu_fd_t *data = (vitastor_qemu_fd_t *)opaque; |
||||
data->callback(data->fd, EPOLLIN); |
||||
} |
||||
|
||||
static void vitastor_c_write_handler(void *opaque) |
||||
{ |
||||
vitastor_qemu_fd_t *data = (vitastor_qemu_fd_t *)opaque; |
||||
data->callback(data->fd, EPOLLOUT); |
||||
} |
||||
|
||||
vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context, |
||||
const char *config_path, const char *etcd_host, const char *etcd_prefix, |
||||
bool use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level) |
||||
{ |
||||
json11::Json cfg_json = vitastor_c_common_config( |
||||
config_path, etcd_host, etcd_prefix, use_rdma, |
||||
rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level |
||||
); |
||||
vitastor_c *self = new vitastor_c; |
||||
self->aio_set_fd_handler = aio_set_fd_handler; |
||||
self->aio_ctx = aio_context; |
||||
self->tfd = new timerfd_manager_t([self](int fd, bool wr, std::function<void(int, int)> callback) |
||||
{ |
||||
if (callback != NULL) |
||||
{ |
||||
self->handlers[fd] = { .fd = fd, .callback = callback }; |
||||
self->aio_set_fd_handler(self->aio_ctx, fd, false, |
||||
vitastor_c_read_handler, wr ? vitastor_c_write_handler : NULL, NULL, &self->handlers[fd]); |
||||
} |
||||
else |
||||
{ |
||||
self->handlers.erase(fd); |
||||
self->aio_set_fd_handler(self->aio_ctx, fd, false, NULL, NULL, NULL, NULL); |
||||
} |
||||
}); |
||||
self->cli = new cluster_client_t(NULL, self->tfd, cfg_json); |
||||
return self; |
||||
} |
||||
|
||||
vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_host, const char *etcd_prefix, |
||||
int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level) |
||||
{ |
||||
json11::Json cfg_json = vitastor_c_common_config( |
||||
config_path, etcd_host, etcd_prefix, use_rdma, |
||||
rdma_device, rdma_port_num, rdma_gid_index, rdma_mtu, log_level |
||||
); |
||||
vitastor_c *self = new vitastor_c; |
||||
self->ringloop = new ring_loop_t(512); |
||||
self->epmgr = new epoll_manager_t(self->ringloop); |
||||
self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json); |
||||
return self; |
||||
} |
||||
|
||||
vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len) |
||||
{ |
||||
json11::Json::object cfg; |
||||
for (int i = 0; i < options_len-1; i += 2) |
||||
{ |
||||
cfg[options[i]] = std::string(options[i+1]); |
||||
} |
||||
json11::Json cfg_json(cfg); |
||||
vitastor_c *self = new vitastor_c; |
||||
self->ringloop = new ring_loop_t(512); |
||||
self->epmgr = new epoll_manager_t(self->ringloop); |
||||
self->cli = new cluster_client_t(self->ringloop, self->epmgr->tfd, cfg_json); |
||||
return self; |
||||
} |
||||
|
||||
void vitastor_c_destroy(vitastor_c *client) |
||||
{ |
||||
delete client->cli; |
||||
if (client->epmgr) |
||||
delete client->epmgr; |
||||
else |
||||
delete client->tfd; |
||||
if (client->ringloop) |
||||
delete client->ringloop; |
||||
delete client; |
||||
} |
||||
|
||||
int vitastor_c_is_ready(vitastor_c *client) |
||||
{ |
||||
return client->cli->is_ready(); |
||||
} |
||||
|
||||
void vitastor_c_uring_wait_ready(vitastor_c *client) |
||||
{ |
||||
while (!client->cli->is_ready()) |
||||
{ |
||||
client->ringloop->loop(); |
||||
if (client->cli->is_ready()) |
||||
break; |
||||
client->ringloop->wait(); |
||||
} |
||||
} |
||||
|
||||
void vitastor_c_uring_handle_events(vitastor_c *client) |
||||
{ |
||||
client->ringloop->loop(); |
||||
} |
||||
|
||||
void vitastor_c_uring_wait_events(vitastor_c *client) |
||||
{ |
||||
client->ringloop->wait(); |
||||
} |
||||
|
||||
static inline void vitastor_c_rw(bool write, vitastor_c *p, uint64_t inode, uint64_t offset, uint64_t len, |
||||
struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque) |
||||
{ |
||||
cluster_op_t *op = new cluster_op_t; |
||||
op->opcode = write ? OSD_OP_WRITE : OSD_OP_READ; |
||||
op->inode = inode; |
||||
op->offset = offset; |
||||
op->len = len; |
||||
for (int i = 0; i < iovcnt; i++) |
||||
{ |
||||
op->iov.push_back(iov[i].iov_base, iov[i].iov_len); |
||||
} |
||||
op->callback = [cb, opaque](cluster_op_t *op) |
||||
{ |
||||
cb(op->retval, opaque); |
||||
delete op; |
||||
}; |
||||
p->cli->execute(op); |
||||
} |
||||
|
||||
void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, |
||||
struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque) |
||||
{ |
||||
vitastor_c_rw(0, client, inode, offset, len, iov, iovcnt, cb, opaque); |
||||
} |
||||
|
||||
void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, |
||||
struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque) |
||||
{ |
||||
vitastor_c_rw(1, client, inode, offset, len, iov, iovcnt, cb, opaque); |
||||
} |
||||
|
||||
void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque) |
||||
{ |
||||
cluster_op_t *op = new cluster_op_t; |
||||
op->opcode = OSD_OP_SYNC; |
||||
op->callback = [cb, opaque](cluster_op_t *op) |
||||
{ |
||||
cb(op->retval, opaque); |
||||
delete op; |
||||
}; |
||||
client->cli->execute(op); |
||||
} |
||||
|
||||
void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler cb, void *opaque) |
||||
{ |
||||
client->cli->on_ready([=]() |
||||
{ |
||||
auto watch = client->cli->st_cli.watch_inode(std::string(image)); |
||||
cb((long)watch, opaque); |
||||
}); |
||||
} |
||||
|
||||
void vitastor_c_close_watch(vitastor_c *client, void *handle) |
||||
{ |
||||
client->cli->st_cli.close_watch((inode_watch_t*)handle); |
||||
} |
||||
|
||||
uint64_t vitastor_c_inode_get_size(void *handle) |
||||
{ |
||||
inode_watch_t *watch = (inode_watch_t*)handle; |
||||
return watch->cfg.size; |
||||
} |
||||
|
||||
uint64_t vitastor_c_inode_get_num(void *handle) |
||||
{ |
||||
inode_watch_t *watch = (inode_watch_t*)handle; |
||||
return watch->cfg.num; |
||||
} |
||||
|
||||
int vitastor_c_inode_get_readonly(void *handle) |
||||
{ |
||||
inode_watch_t *watch = (inode_watch_t*)handle; |
||||
return watch->cfg.readonly; |
||||
} |
||||
|
||||
} |
||||
@ -0,0 +1,54 @@ |
||||
// Copyright (c) Vitaliy Filippov, 2019+
|
||||
// License: VNPL-1.1 or GNU GPL-2.0+ (see README.md for details)
|
||||
|
||||
// Simplified C client library for QEMU, fio and other external drivers
|
||||
|
||||
#ifndef VITASTOR_QEMU_PROXY_H |
||||
#define VITASTOR_QEMU_PROXY_H |
||||
|
||||
#ifndef POOL_ID_BITS |
||||
#define POOL_ID_BITS 16 |
||||
#endif |
||||
#include <stdint.h> |
||||
#include <sys/uio.h> |
||||
|
||||
#ifdef __cplusplus |
||||
extern "C" { |
||||
#endif |
||||
|
||||
struct vitastor_c; |
||||
typedef struct vitastor_c vitastor_c; |
||||
|
||||
typedef void VitastorIOHandler(long retval, void *opaque); |
||||
|
||||
// QEMU
|
||||
typedef void IOHandler(void *opaque); |
||||
typedef void QEMUSetFDHandler(void *ctx, int fd, int is_external, IOHandler *fd_read, IOHandler *fd_write, void *poll_fn, void *opaque); |
||||
|
||||
vitastor_c *vitastor_c_create_qemu(QEMUSetFDHandler *aio_set_fd_handler, void *aio_context, |
||||
const char *config_path, const char *etcd_host, const char *etcd_prefix, |
||||
bool use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level); |
||||
vitastor_c *vitastor_c_create_uring(const char *config_path, const char *etcd_host, const char *etcd_prefix, |
||||
int use_rdma, const char *rdma_device, int rdma_port_num, int rdma_gid_index, int rdma_mtu, int log_level); |
||||
vitastor_c *vitastor_c_create_uring_json(const char **options, int options_len); |
||||
void vitastor_c_destroy(vitastor_c *client); |
||||
int vitastor_c_is_ready(vitastor_c *client); |
||||
void vitastor_c_uring_wait_ready(vitastor_c *client); |
||||
void vitastor_c_uring_handle_events(vitastor_c *client); |
||||
void vitastor_c_uring_wait_events(vitastor_c *client); |
||||
void vitastor_c_read(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, |
||||
struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque); |
||||
void vitastor_c_write(vitastor_c *client, uint64_t inode, uint64_t offset, uint64_t len, |
||||
struct iovec *iov, int iovcnt, VitastorIOHandler cb, void *opaque); |
||||
void vitastor_c_sync(vitastor_c *client, VitastorIOHandler cb, void *opaque); |
||||
void vitastor_c_watch_inode(vitastor_c *client, char *image, VitastorIOHandler cb, void *opaque); |
||||
void vitastor_c_close_watch(vitastor_c *client, void *handle); |
||||
uint64_t vitastor_c_inode_get_size(void *handle); |
||||
uint64_t vitastor_c_inode_get_num(void *handle); |
||||
int vitastor_c_inode_get_readonly(void *handle); |
||||
|
||||
#ifdef __cplusplus |
||||
} |
||||
#endif |
||||
|
||||
#endif |
||||
Loading…
Reference in new issue