Slightly cleanup socket send/receive code
parent
b466e215f0
commit
5feff1ffb9
|
@ -199,10 +199,10 @@ struct cluster_client_t
|
||||||
|
|
||||||
bool try_send(osd_client_t & cl);
|
bool try_send(osd_client_t & cl);
|
||||||
void send_replies();
|
void send_replies();
|
||||||
void handle_send(ring_data_t *data, int peer_fd);
|
void handle_send(int result, int peer_fd);
|
||||||
|
|
||||||
void read_requests();
|
void read_requests();
|
||||||
void handle_read(ring_data_t *data, int peer_fd);
|
bool handle_read(int result, int peer_fd);
|
||||||
void handle_finished_read(osd_client_t & cl);
|
void handle_finished_read(osd_client_t & cl);
|
||||||
void handle_op_hdr(osd_client_t *cl);
|
void handle_op_hdr(osd_client_t *cl);
|
||||||
void handle_reply_hdr(osd_client_t *cl);
|
void handle_reply_hdr(osd_client_t *cl);
|
||||||
|
|
|
@ -52,6 +52,7 @@ struct http_co_t
|
||||||
|
|
||||||
~http_co_t();
|
~http_co_t();
|
||||||
void start_connection();
|
void start_connection();
|
||||||
|
void handle_events();
|
||||||
void handle_connect_result();
|
void handle_connect_result();
|
||||||
void submit_read();
|
void submit_read();
|
||||||
void submit_send();
|
void submit_send();
|
||||||
|
@ -142,6 +143,7 @@ void websocket_t::close()
|
||||||
|
|
||||||
http_co_t::~http_co_t()
|
http_co_t::~http_co_t()
|
||||||
{
|
{
|
||||||
|
epoll_events = 0;
|
||||||
if (timeout_id >= 0)
|
if (timeout_id >= 0)
|
||||||
{
|
{
|
||||||
tfd->clear_timer(timeout_id);
|
tfd->clear_timer(timeout_id);
|
||||||
|
@ -204,25 +206,6 @@ void http_co_t::start_connection()
|
||||||
delete this;
|
delete this;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
tfd->set_fd_handler(peer_fd, [this](int peer_fd, int epoll_events)
|
|
||||||
{
|
|
||||||
this->epoll_events |= epoll_events;
|
|
||||||
if (state == HTTP_CO_CONNECTING)
|
|
||||||
{
|
|
||||||
handle_connect_result();
|
|
||||||
}
|
|
||||||
else
|
|
||||||
{
|
|
||||||
if (this->epoll_events & EPOLLIN)
|
|
||||||
{
|
|
||||||
submit_read();
|
|
||||||
}
|
|
||||||
else if (this->epoll_events & (EPOLLRDHUP|EPOLLERR))
|
|
||||||
{
|
|
||||||
delete this;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
});
|
|
||||||
epoll_events = 0;
|
epoll_events = 0;
|
||||||
// Finally call connect
|
// Finally call connect
|
||||||
r = ::connect(peer_fd, (sockaddr*)&addr, sizeof(addr));
|
r = ::connect(peer_fd, (sockaddr*)&addr, sizeof(addr));
|
||||||
|
@ -232,12 +215,39 @@ void http_co_t::start_connection()
|
||||||
delete this;
|
delete this;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
tfd->set_fd_handler(peer_fd, [this](int peer_fd, int epoll_events)
|
||||||
|
{
|
||||||
|
this->epoll_events |= epoll_events;
|
||||||
|
handle_events();
|
||||||
|
});
|
||||||
state = HTTP_CO_CONNECTING;
|
state = HTTP_CO_CONNECTING;
|
||||||
}
|
}
|
||||||
|
|
||||||
void http_co_t::handle_connect_result()
|
void http_co_t::handle_events()
|
||||||
{
|
{
|
||||||
if (epoll_events & (EPOLLOUT | EPOLLERR))
|
while (epoll_events)
|
||||||
|
{
|
||||||
|
if (state == HTTP_CO_CONNECTING)
|
||||||
|
{
|
||||||
|
handle_connect_result();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
epoll_events &= ~EPOLLOUT;
|
||||||
|
if (epoll_events & EPOLLIN)
|
||||||
|
{
|
||||||
|
submit_read();
|
||||||
|
}
|
||||||
|
else if (epoll_events & (EPOLLRDHUP|EPOLLERR))
|
||||||
|
{
|
||||||
|
delete this;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void http_co_t::handle_connect_result()
|
||||||
{
|
{
|
||||||
int result = 0;
|
int result = 0;
|
||||||
socklen_t result_len = sizeof(result);
|
socklen_t result_len = sizeof(result);
|
||||||
|
@ -256,16 +266,10 @@ void http_co_t::handle_connect_result()
|
||||||
state = HTTP_CO_SENDING_REQUEST;
|
state = HTTP_CO_SENDING_REQUEST;
|
||||||
submit_send();
|
submit_send();
|
||||||
}
|
}
|
||||||
else
|
|
||||||
{
|
|
||||||
delete this;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void http_co_t::submit_read()
|
void http_co_t::submit_read()
|
||||||
{
|
{
|
||||||
int res;
|
int res;
|
||||||
again:
|
|
||||||
if (rbuf.size() != READ_BUFFER_SIZE)
|
if (rbuf.size() != READ_BUFFER_SIZE)
|
||||||
{
|
{
|
||||||
rbuf.resize(READ_BUFFER_SIZE);
|
rbuf.resize(READ_BUFFER_SIZE);
|
||||||
|
@ -273,34 +277,24 @@ again:
|
||||||
read_iov = { .iov_base = rbuf.data(), .iov_len = READ_BUFFER_SIZE };
|
read_iov = { .iov_base = rbuf.data(), .iov_len = READ_BUFFER_SIZE };
|
||||||
read_msg.msg_iov = &read_iov;
|
read_msg.msg_iov = &read_iov;
|
||||||
read_msg.msg_iovlen = 1;
|
read_msg.msg_iovlen = 1;
|
||||||
epoll_events = epoll_events & ~EPOLLIN;
|
|
||||||
res = recvmsg(peer_fd, &read_msg, 0);
|
res = recvmsg(peer_fd, &read_msg, 0);
|
||||||
if (res < 0)
|
if (res < 0)
|
||||||
{
|
{
|
||||||
res = -errno;
|
res = -errno;
|
||||||
}
|
}
|
||||||
if (res == -EAGAIN)
|
if (res == -EAGAIN || res == 0)
|
||||||
{
|
{
|
||||||
res = 0;
|
epoll_events = epoll_events & ~EPOLLIN;
|
||||||
}
|
}
|
||||||
if (res < 0)
|
else if (res < 0)
|
||||||
{
|
{
|
||||||
delete this;
|
delete this;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
else if (res > 0)
|
||||||
|
{
|
||||||
response += std::string(rbuf.data(), res);
|
response += std::string(rbuf.data(), res);
|
||||||
if (res == READ_BUFFER_SIZE)
|
handle_read();
|
||||||
{
|
|
||||||
goto again;
|
|
||||||
}
|
|
||||||
if (!handle_read())
|
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (res < READ_BUFFER_SIZE && (epoll_events & (EPOLLRDHUP|EPOLLERR)))
|
|
||||||
{
|
|
||||||
delete this;
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -331,7 +325,9 @@ again:
|
||||||
if (state == HTTP_CO_SENDING_REQUEST)
|
if (state == HTTP_CO_SENDING_REQUEST)
|
||||||
{
|
{
|
||||||
if (sent >= request.size())
|
if (sent >= request.size())
|
||||||
|
{
|
||||||
state = HTTP_CO_REQUEST_SENT;
|
state = HTTP_CO_REQUEST_SENT;
|
||||||
|
}
|
||||||
else
|
else
|
||||||
goto again;
|
goto again;
|
||||||
}
|
}
|
||||||
|
|
27
osd.cpp
27
osd.cpp
|
@ -283,6 +283,22 @@ restart:
|
||||||
for (int i = 0; i < nfds; i++)
|
for (int i = 0; i < nfds; i++)
|
||||||
{
|
{
|
||||||
if (events[i].data.fd == listen_fd)
|
if (events[i].data.fd == listen_fd)
|
||||||
|
{
|
||||||
|
accept_connections();
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
auto & cb = epoll_handlers[events[i].data.fd];
|
||||||
|
cb(events[i].data.fd, events[i].events);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (nfds == MAX_EPOLL_EVENTS)
|
||||||
|
{
|
||||||
|
goto restart;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void osd_t::accept_connections()
|
||||||
{
|
{
|
||||||
// Accept new connections
|
// Accept new connections
|
||||||
sockaddr_in addr;
|
sockaddr_in addr;
|
||||||
|
@ -317,17 +333,6 @@ restart:
|
||||||
throw std::runtime_error(std::string("accept: ") + strerror(errno));
|
throw std::runtime_error(std::string("accept: ") + strerror(errno));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
|
||||||
{
|
|
||||||
auto & cb = epoll_handlers[events[i].data.fd];
|
|
||||||
cb(events[i].data.fd, events[i].events);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (nfds == MAX_EPOLL_EVENTS)
|
|
||||||
{
|
|
||||||
goto restart;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
void osd_t::exec_op(osd_op_t *cur_op)
|
void osd_t::exec_op(osd_op_t *cur_op)
|
||||||
{
|
{
|
||||||
|
|
1
osd.h
1
osd.h
|
@ -149,6 +149,7 @@ class osd_t
|
||||||
|
|
||||||
// event loop, socket read/write
|
// event loop, socket read/write
|
||||||
void loop();
|
void loop();
|
||||||
|
void accept_connections();
|
||||||
void set_fd_handler(int fd, std::function<void(int, int)> handler);
|
void set_fd_handler(int fd, std::function<void(int, int)> handler);
|
||||||
void handle_epoll_events();
|
void handle_epoll_events();
|
||||||
|
|
||||||
|
|
|
@ -25,26 +25,26 @@ void cluster_client_t::read_requests()
|
||||||
}
|
}
|
||||||
cl.read_msg.msg_iov = &cl.read_iov;
|
cl.read_msg.msg_iov = &cl.read_iov;
|
||||||
cl.read_msg.msg_iovlen = 1;
|
cl.read_msg.msg_iovlen = 1;
|
||||||
data->callback = [this, peer_fd](ring_data_t *data) { handle_read(data, peer_fd); };
|
data->callback = [this, peer_fd](ring_data_t *data) { handle_read(data->res, peer_fd); };
|
||||||
my_uring_prep_recvmsg(sqe, peer_fd, &cl.read_msg, 0);
|
my_uring_prep_recvmsg(sqe, peer_fd, &cl.read_msg, 0);
|
||||||
}
|
}
|
||||||
read_ready_clients.clear();
|
read_ready_clients.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
void cluster_client_t::handle_read(ring_data_t *data, int peer_fd)
|
bool cluster_client_t::handle_read(int result, int peer_fd)
|
||||||
{
|
{
|
||||||
auto cl_it = clients.find(peer_fd);
|
auto cl_it = clients.find(peer_fd);
|
||||||
if (cl_it != clients.end())
|
if (cl_it != clients.end())
|
||||||
{
|
{
|
||||||
auto & cl = cl_it->second;
|
auto & cl = cl_it->second;
|
||||||
if (data->res < 0 && data->res != -EAGAIN)
|
if (result < 0 && result != -EAGAIN)
|
||||||
{
|
{
|
||||||
// this is a client socket, so don't panic. just disconnect it
|
// this is a client socket, so don't panic. just disconnect it
|
||||||
printf("Client %d socket read error: %d (%s). Disconnecting client\n", peer_fd, -data->res, strerror(-data->res));
|
printf("Client %d socket read error: %d (%s). Disconnecting client\n", peer_fd, -result, strerror(-result));
|
||||||
stop_client(peer_fd);
|
stop_client(peer_fd);
|
||||||
return;
|
return false;
|
||||||
}
|
}
|
||||||
if (data->res == -EAGAIN || cl.read_iov.iov_base == cl.in_buf && data->res < receive_buffer_size)
|
if (result == -EAGAIN || result < cl.read_iov.iov_len)
|
||||||
{
|
{
|
||||||
cl.read_ready--;
|
cl.read_ready--;
|
||||||
if (cl.read_ready > 0)
|
if (cl.read_ready > 0)
|
||||||
|
@ -54,16 +54,12 @@ void cluster_client_t::handle_read(ring_data_t *data, int peer_fd)
|
||||||
{
|
{
|
||||||
read_ready_clients.push_back(peer_fd);
|
read_ready_clients.push_back(peer_fd);
|
||||||
}
|
}
|
||||||
if (data->res == -EAGAIN)
|
if (result > 0)
|
||||||
{
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
if (data->res > 0)
|
|
||||||
{
|
{
|
||||||
if (cl.read_iov.iov_base == cl.in_buf)
|
if (cl.read_iov.iov_base == cl.in_buf)
|
||||||
{
|
{
|
||||||
// Compose operation(s) from the buffer
|
// Compose operation(s) from the buffer
|
||||||
int remain = data->res;
|
int remain = result;
|
||||||
void *curbuf = cl.in_buf;
|
void *curbuf = cl.in_buf;
|
||||||
while (remain > 0)
|
while (remain > 0)
|
||||||
{
|
{
|
||||||
|
@ -99,16 +95,21 @@ void cluster_client_t::handle_read(ring_data_t *data, int peer_fd)
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// Long data
|
// Long data
|
||||||
cl.read_remaining -= data->res;
|
cl.read_remaining -= result;
|
||||||
cl.read_buf += data->res;
|
cl.read_buf += result;
|
||||||
if (cl.read_remaining <= 0)
|
if (cl.read_remaining <= 0)
|
||||||
{
|
{
|
||||||
handle_finished_read(cl);
|
handle_finished_read(cl);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if (result >= cl.read_iov.iov_len)
|
||||||
|
{
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
void cluster_client_t::handle_finished_read(osd_client_t & cl)
|
void cluster_client_t::handle_finished_read(osd_client_t & cl)
|
||||||
{
|
{
|
||||||
|
|
20
osd_send.cpp
20
osd_send.cpp
|
@ -65,7 +65,7 @@ bool cluster_client_t::try_send(osd_client_t & cl)
|
||||||
}
|
}
|
||||||
cl.write_msg.msg_iov = cl.write_op->send_list.get_iovec();
|
cl.write_msg.msg_iov = cl.write_op->send_list.get_iovec();
|
||||||
cl.write_msg.msg_iovlen = cl.write_op->send_list.get_size();
|
cl.write_msg.msg_iovlen = cl.write_op->send_list.get_size();
|
||||||
data->callback = [this, peer_fd](ring_data_t *data) { handle_send(data, peer_fd); };
|
data->callback = [this, peer_fd](ring_data_t *data) { handle_send(data->res, peer_fd); };
|
||||||
my_uring_prep_sendmsg(sqe, peer_fd, &cl.write_msg, 0);
|
my_uring_prep_sendmsg(sqe, peer_fd, &cl.write_msg, 0);
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
@ -84,34 +84,34 @@ void cluster_client_t::send_replies()
|
||||||
write_ready_clients.clear();
|
write_ready_clients.clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
void cluster_client_t::handle_send(ring_data_t *data, int peer_fd)
|
void cluster_client_t::handle_send(int result, int peer_fd)
|
||||||
{
|
{
|
||||||
auto cl_it = clients.find(peer_fd);
|
auto cl_it = clients.find(peer_fd);
|
||||||
if (cl_it != clients.end())
|
if (cl_it != clients.end())
|
||||||
{
|
{
|
||||||
auto & cl = cl_it->second;
|
auto & cl = cl_it->second;
|
||||||
if (data->res < 0 && data->res != -EAGAIN)
|
if (result < 0 && result != -EAGAIN)
|
||||||
{
|
{
|
||||||
// this is a client socket, so don't panic. just disconnect it
|
// this is a client socket, so don't panic. just disconnect it
|
||||||
printf("Client %d socket write error: %d (%s). Disconnecting client\n", peer_fd, -data->res, strerror(-data->res));
|
printf("Client %d socket write error: %d (%s). Disconnecting client\n", peer_fd, -result, strerror(-result));
|
||||||
stop_client(peer_fd);
|
stop_client(peer_fd);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
if (data->res >= 0)
|
if (result >= 0)
|
||||||
{
|
{
|
||||||
osd_op_t *cur_op = cl.write_op;
|
osd_op_t *cur_op = cl.write_op;
|
||||||
while (data->res > 0 && cur_op->send_list.sent < cur_op->send_list.count)
|
while (result > 0 && cur_op->send_list.sent < cur_op->send_list.count)
|
||||||
{
|
{
|
||||||
iovec & iov = cur_op->send_list.buf[cur_op->send_list.sent];
|
iovec & iov = cur_op->send_list.buf[cur_op->send_list.sent];
|
||||||
if (iov.iov_len <= data->res)
|
if (iov.iov_len <= result)
|
||||||
{
|
{
|
||||||
data->res -= iov.iov_len;
|
result -= iov.iov_len;
|
||||||
cur_op->send_list.sent++;
|
cur_op->send_list.sent++;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
iov.iov_len -= data->res;
|
iov.iov_len -= result;
|
||||||
iov.iov_base += data->res;
|
iov.iov_base += result;
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue