diff --git a/messenger.cpp b/messenger.cpp index 8a0663ac..d5512bee 100644 --- a/messenger.cpp +++ b/messenger.cpp @@ -222,7 +222,7 @@ void osd_messenger_t::on_connect_peer(osd_num_t peer_osd, int peer_fd) } if (log_level > 0) { - printf("[OSD %lu] Connected with peer OSD %lu (fd %d)\n", osd_num, peer_osd, peer_fd); + printf("[OSD %lu] Connected with peer OSD %lu (client %d)\n", osd_num, peer_osd, peer_fd); } wanted_peers.erase(peer_osd); repeer_pgs(peer_osd); diff --git a/msgr_send.cpp b/msgr_send.cpp index 50c1647a..54ba72f4 100644 --- a/msgr_send.cpp +++ b/msgr_send.cpp @@ -58,6 +58,7 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op) to_outbox.push_back(NULL); for (int i = 0; i < cur_op->iov.count; i++) { + assert(cur_op->iov.buf[i].iov_base); to_send_list.push_back(cur_op->iov.buf[i]); to_outbox.push_back(i == cur_op->iov.count-1 ? cur_op : NULL); } diff --git a/osd_rmw.cpp b/osd_rmw.cpp index 886de5fd..818e2785 100644 --- a/osd_rmw.cpp +++ b/osd_rmw.cpp @@ -193,6 +193,16 @@ void* calc_rmw(void *request_buf, osd_rmw_stripe_t *stripes, uint64_t *read_osd_ if (write_osd_set[role] != 0) { write_parity = 1; + if (write_osd_set[role] != read_osd_set[role]) + { + start = 0; + end = chunk_size; + for (int r2 = pg_minsize; r2 < role; r2++) + { + stripes[r2].write_start = start; + stripes[r2].write_end = end; + } + } stripes[role].write_start = start; stripes[role].write_end = end; } @@ -208,9 +218,9 @@ void* calc_rmw(void *request_buf, osd_rmw_stripe_t *stripes, uint64_t *read_osd_ { pg_cursize = 0; // Object is degraded/misplaced and will be moved to - for (int role = 0; role < pg_minsize; role++) + for (int role = 0; role < pg_size; role++) { - if (write_osd_set[role] != read_osd_set[role] && write_osd_set[role] != 0) + if (role < pg_minsize && write_osd_set[role] != read_osd_set[role] && write_osd_set[role] != 0) { // We need to get data for any moved / recovered chunk // And we need a continuous write buffer so we'll only optimize @@ -228,20 +238,6 @@ void* calc_rmw(void *request_buf, osd_rmw_stripe_t *stripes, uint64_t *read_osd_ pg_cursize++; } } - for (int role = pg_minsize; role < pg_size; role++) - { - if (write_osd_set[role] != read_osd_set[role] && write_osd_set[role] != 0) - { - for (int r2 = 0; r2 < pg_minsize; r2++) - { - cover_read(0, chunk_size, stripes[r2]); - } - } - if (read_osd_set[role] != 0) - { - pg_cursize++; - } - } } if (pg_cursize < pg_size) { @@ -397,6 +393,14 @@ void calc_rmw_parity_xor(osd_rmw_stripe_t *stripes, int pg_size, uint64_t *read_ end = std::max(stripes[role].req_end, end); } } + for (int role = pg_minsize; role < pg_size; role++) + { + if (write_osd_set[role] != 0 && write_osd_set[role] != read_osd_set[role]) + { + start = 0; + end = chunk_size; + } + } } if (write_osd_set != read_osd_set) { @@ -463,4 +467,20 @@ void calc_rmw_parity_xor(osd_rmw_stripe_t *stripes, int pg_size, uint64_t *read_ } } } +#ifdef RMW_DEBUG + printf("calc_rmw_xor:\n"); + for (int role = 0; role < pg_size; role++) + { + auto & s = stripes[role]; + printf( + "Tr=%lu Tw=%lu Q=%x-%x R=%x-%x W=%x-%x Rb=%lx Wb=%lx\n", + read_osd_set[role], write_osd_set[role], + s.req_start, s.req_end, + s.read_start, s.read_end, + s.write_start, s.write_end, + (uint64_t)s.read_buf, + (uint64_t)s.write_buf + ); + } +#endif } diff --git a/osd_rmw_test.cpp b/osd_rmw_test.cpp index 84f9dc19..5e70a4d1 100644 --- a/osd_rmw_test.cpp +++ b/osd_rmw_test.cpp @@ -1,6 +1,8 @@ // Copyright (c) Vitaliy Filippov, 2019+ // License: VNPL-1.0 (see README.md for details) +#define RMW_DEBUG + #include #include "osd_rmw.cpp" #include "test_pattern.h" @@ -15,6 +17,7 @@ void test8(); void test9(); void test10(); void test11(); +void test12(); /*** @@ -137,6 +140,8 @@ int main(int narg, char *args[]) test10(); // Test 11 test11(); + // Test 12 + test12(); // End printf("all ok\n"); return 0; @@ -471,3 +476,42 @@ void test11() free(rmw_buf); free(write_buf); } + +void test12() +{ + osd_num_t osd_set[3] = { 1, 2, 0 }; + osd_num_t write_osd_set[3] = { 1, 2, 3 }; + osd_rmw_stripe_t stripes[3] = { 0 }; + // Test 11.0 + split_stripes(2, 128*1024, 0, 0, stripes); + assert(stripes[0].req_start == 0 && stripes[0].req_end == 0); + assert(stripes[1].req_start == 0 && stripes[1].req_end == 0); + assert(stripes[2].req_start == 0 && stripes[2].req_end == 0); + // Test 11.1 + void *rmw_buf = calc_rmw(NULL, stripes, osd_set, 3, 2, 3, write_osd_set, 128*1024); + assert(rmw_buf); + assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024); + assert(stripes[1].read_start == 0 && stripes[1].read_end == 128*1024); + assert(stripes[2].read_start == 0 && stripes[2].read_end == 0); + assert(stripes[0].write_start == 0 && stripes[0].write_end == 0); + assert(stripes[1].write_start == 0 && stripes[1].write_end == 0); + assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024); + assert(stripes[0].read_buf == rmw_buf+128*1024); + assert(stripes[1].read_buf == rmw_buf+2*128*1024); + assert(stripes[2].read_buf == NULL); + assert(stripes[0].write_buf == NULL); + assert(stripes[1].write_buf == NULL); + assert(stripes[2].write_buf == rmw_buf); + // Test 11.2 + set_pattern(stripes[0].read_buf, 128*1024, PATTERN1); + set_pattern(stripes[1].read_buf, 128*1024, PATTERN2); + calc_rmw_parity_xor(stripes, 3, osd_set, write_osd_set, 128*1024); + assert(stripes[0].write_start == 0 && stripes[0].write_end == 0); + assert(stripes[1].write_start == 0 && stripes[1].write_end == 0); + assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024); + assert(stripes[0].write_buf == NULL); + assert(stripes[1].write_buf == NULL); + assert(stripes[2].write_buf == rmw_buf); + check_pattern(stripes[2].write_buf, 128*1024, PATTERN1^PATTERN2); + free(rmw_buf); +}