vitalif
/
vitastor
5
4
Fork 9
Code Issues 1 Pull Requests 2 Actions Releases 68 Wiki Activity

Reuse incoming RDMA buffers

csi-use-vitastor-cli
Vitaliy Filippov 2023-02-26 00:55:01 +03:00
parent 8139a34e97
commit 138ffe4032
2 changed files with 13 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
src/msgr_rdma.cpp
View File

@ -410,13 +410,18 @@ bool osd_messenger_t::try_send_rdma(osd_client_t *cl)
return true; return true;
} }
static void try_recv_rdma_wr(osd_client_t *cl, ibv_sge *sge, int op_sge) static void try_recv_rdma_wr(osd_client_t *cl, void *buf)
{ {
ibv_sge sge = {
.addr = (uintptr_t)buf,
.length = (uint32_t)cl->rdma_conn->max_msg,
.lkey = cl->rdma_conn->ctx->mr->lkey,
};
ibv_recv_wr *bad_wr = NULL; ibv_recv_wr *bad_wr = NULL;
ibv_recv_wr wr = { ibv_recv_wr wr = {
.wr_id = (uint64_t)(cl->peer_fd*2), .wr_id = (uint64_t)(cl->peer_fd*2),
.sg_list = sge, .sg_list = &sge,
.num_sge = op_sge, .num_sge = 1,
}; };
int err = ibv_post_recv(cl->rdma_conn->qp, &wr, &bad_wr); int err = ibv_post_recv(cl->rdma_conn->qp, &wr, &bad_wr);
if (err || bad_wr) if (err || bad_wr)
@ -434,12 +439,7 @@ bool osd_messenger_t::try_recv_rdma(osd_client_t *cl)
{ {
void *buf = malloc_or_die(rc->max_msg); void *buf = malloc_or_die(rc->max_msg);
rc->recv_buffers.push_back(buf); rc->recv_buffers.push_back(buf);
ibv_sge sge = { try_recv_rdma_wr(cl, buf);
.addr = (uintptr_t)buf,
.length = (uint32_t)rc->max_msg,
.lkey = rc->ctx->mr->lkey,
};
try_recv_rdma_wr(cl, &sge, 1);
} }
return true; return true;
} }
@ -490,14 +490,13 @@ void osd_messenger_t::handle_rdma_events()
if (!is_send) if (!is_send)
{ {
cl->rdma_conn->cur_recv--; cl->rdma_conn->cur_recv--;
if (!handle_read_buffer(cl, cl->rdma_conn->recv_buffers[0], wc[i].byte_len)) if (!handle_read_buffer(cl, cl->rdma_conn->recv_buffers[cl->rdma_conn->next_recv_buf], wc[i].byte_len))
{ {
// handle_read_buffer may stop the client // handle_read_buffer may stop the client
continue; continue;
} }
free(cl->rdma_conn->recv_buffers[0]); try_recv_rdma_wr(cl, cl->rdma_conn->recv_buffers[cl->rdma_conn->next_recv_buf]);
cl->rdma_conn->recv_buffers.erase(cl->rdma_conn->recv_buffers.begin(), cl->rdma_conn->recv_buffers.begin()+1); cl->rdma_conn->next_recv_buf = (cl->rdma_conn->next_recv_buf+1) % cl->rdma_conn->recv_buffers.size();
try_recv_rdma(cl);
} }
else else
{ {

2
src/msgr_rdma.h
View File

@ -49,7 +49,7 @@ struct msgr_rdma_connection_t
uint64_t max_msg = 0; uint64_t max_msg = 0;
int send_pos = 0, send_buf_pos = 0; int send_pos = 0, send_buf_pos = 0;
int recv_pos = 0, recv_buf_pos = 0; int next_recv_buf = 0;
std::vector<void*> recv_buffers; std::vector<void*> recv_buffers;
~msgr_rdma_connection_t(); ~msgr_rdma_connection_t();