Fix NULL rmw buffer after the latest changes and add a testcase for it

messenger.cpp

@@ -222,7 +222,7 @@ void osd_messenger_t::on_connect_peer(osd_num_t peer_osd, int peer_fd)
     }
     if (log_level > 0)
     {
-        printf("[OSD %lu] Connected with peer OSD %lu (fd %d)\n", osd_num, peer_osd, peer_fd);
+        printf("[OSD %lu] Connected with peer OSD %lu (client %d)\n", osd_num, peer_osd, peer_fd);
     }
     wanted_peers.erase(peer_osd);
     repeer_pgs(peer_osd);

msgr_send.cpp

@@ -58,6 +58,7 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
         to_outbox.push_back(NULL);
         for (int i = 0; i < cur_op->iov.count; i++)
         {
+            assert(cur_op->iov.buf[i].iov_base);
             to_send_list.push_back(cur_op->iov.buf[i]);
             to_outbox.push_back(i == cur_op->iov.count-1 ? cur_op : NULL);
         }

osd_rmw.cpp

@@ -193,6 +193,16 @@ void* calc_rmw(void *request_buf, osd_rmw_stripe_t *stripes, uint64_t *read_osd_
         if (write_osd_set[role] != 0)
         {
             write_parity = 1;
+            if (write_osd_set[role] != read_osd_set[role])
+            {
+                start = 0;
+                end = chunk_size;
+                for (int r2 = pg_minsize; r2 < role; r2++)
+                {
+                    stripes[r2].write_start = start;
+                    stripes[r2].write_end = end;
+                }
+            }
             stripes[role].write_start = start;
             stripes[role].write_end = end;
         }
@@ -208,9 +218,9 @@ void* calc_rmw(void *request_buf, osd_rmw_stripe_t *stripes, uint64_t *read_osd_
     {
         pg_cursize = 0;
         // Object is degraded/misplaced and will be moved to <write_osd_set>
-        for (int role = 0; role < pg_minsize; role++)
+        for (int role = 0; role < pg_size; role++)
         {
-            if (write_osd_set[role] != read_osd_set[role] && write_osd_set[role] != 0)
+            if (role < pg_minsize && write_osd_set[role] != read_osd_set[role] && write_osd_set[role] != 0)
             {
                 // We need to get data for any moved / recovered chunk
                 // And we need a continuous write buffer so we'll only optimize
@@ -228,20 +238,6 @@ void* calc_rmw(void *request_buf, osd_rmw_stripe_t *stripes, uint64_t *read_osd_
                 pg_cursize++;
             }
         }
-        for (int role = pg_minsize; role < pg_size; role++)
-        {
-            if (write_osd_set[role] != read_osd_set[role] && write_osd_set[role] != 0)
-            {
-                for (int r2 = 0; r2 < pg_minsize; r2++)
-                {
-                    cover_read(0, chunk_size, stripes[r2]);
-                }
-            }
-            if (read_osd_set[role] != 0)
-            {
-                pg_cursize++;
-            }
-        }
     }
     if (pg_cursize < pg_size)
     {
@@ -397,6 +393,14 @@ void calc_rmw_parity_xor(osd_rmw_stripe_t *stripes, int pg_size, uint64_t *read_
                 end = std::max(stripes[role].req_end, end);
             }
         }
+        for (int role = pg_minsize; role < pg_size; role++)
+        {
+            if (write_osd_set[role] != 0 && write_osd_set[role] != read_osd_set[role])
+            {
+                start = 0;
+                end = chunk_size;
+            }
+        }
     }
     if (write_osd_set != read_osd_set)
     {
@@ -463,4 +467,20 @@ void calc_rmw_parity_xor(osd_rmw_stripe_t *stripes, int pg_size, uint64_t *read_
             }
         }
     }
+#ifdef RMW_DEBUG
+    printf("calc_rmw_xor:\n");
+    for (int role = 0; role < pg_size; role++)
+    {
+        auto & s = stripes[role];
+        printf(
+            "Tr=%lu Tw=%lu Q=%x-%x R=%x-%x W=%x-%x Rb=%lx Wb=%lx\n",
+            read_osd_set[role], write_osd_set[role],
+            s.req_start, s.req_end,
+            s.read_start, s.read_end,
+            s.write_start, s.write_end,
+            (uint64_t)s.read_buf,
+            (uint64_t)s.write_buf
+        );
+    }
+#endif
 }

osd_rmw_test.cpp

@@ -1,6 +1,8 @@
 // Copyright (c) Vitaliy Filippov, 2019+
 // License: VNPL-1.0 (see README.md for details)
 
+#define RMW_DEBUG
+
 #include <string.h>
 #include "osd_rmw.cpp"
 #include "test_pattern.h"
@@ -15,6 +17,7 @@ void test8();
 void test9();
 void test10();
 void test11();
+void test12();
 
 /***
 
@@ -137,6 +140,8 @@ int main(int narg, char *args[])
     test10();
     // Test 11
     test11();
+    // Test 12
+    test12();
     // End
     printf("all ok\n");
     return 0;
@@ -471,3 +476,42 @@ void test11()
     free(rmw_buf);
     free(write_buf);
 }
+
+void test12()
+{
+    osd_num_t osd_set[3] = { 1, 2, 0 };
+    osd_num_t write_osd_set[3] = { 1, 2, 3 };
+    osd_rmw_stripe_t stripes[3] = { 0 };
+    // Test 11.0
+    split_stripes(2, 128*1024, 0, 0, stripes);
+    assert(stripes[0].req_start == 0 && stripes[0].req_end == 0);
+    assert(stripes[1].req_start == 0 && stripes[1].req_end == 0);
+    assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
+    // Test 11.1
+    void *rmw_buf = calc_rmw(NULL, stripes, osd_set, 3, 2, 3, write_osd_set, 128*1024);
+    assert(rmw_buf);
+    assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024);
+    assert(stripes[1].read_start == 0 && stripes[1].read_end == 128*1024);
+    assert(stripes[2].read_start == 0 && stripes[2].read_end == 0);
+    assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
+    assert(stripes[1].write_start == 0 && stripes[1].write_end == 0);
+    assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
+    assert(stripes[0].read_buf == rmw_buf+128*1024);
+    assert(stripes[1].read_buf == rmw_buf+2*128*1024);
+    assert(stripes[2].read_buf == NULL);
+    assert(stripes[0].write_buf == NULL);
+    assert(stripes[1].write_buf == NULL);
+    assert(stripes[2].write_buf == rmw_buf);
+    // Test 11.2
+    set_pattern(stripes[0].read_buf, 128*1024, PATTERN1);
+    set_pattern(stripes[1].read_buf, 128*1024, PATTERN2);
+    calc_rmw_parity_xor(stripes, 3, osd_set, write_osd_set, 128*1024);
+    assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
+    assert(stripes[1].write_start == 0 && stripes[1].write_end == 0);
+    assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
+    assert(stripes[0].write_buf == NULL);
+    assert(stripes[1].write_buf == NULL);
+    assert(stripes[2].write_buf == rmw_buf);
+    check_pattern(stripes[2].write_buf, 128*1024, PATTERN1^PATTERN2);
+    free(rmw_buf);
+}