From 9038d423279af0570647b01dc6aaefdebd48fada Mon Sep 17 00:00:00 2001
From: Vitaliy Filippov <vitalif@yourcmc.ru>
Date: Mon, 1 Feb 2021 01:37:54 +0300
Subject: [PATCH] Fix several snapshot I/O bugs

---
 src/msgr_receive.cpp       |  2 +-
 src/msgr_send.cpp          | 33 +++++++++++++++++++++------------
 src/osd_primary.cpp        |  5 +++--
 src/osd_primary_subops.cpp |  2 +-
 src/osd_secondary.cpp      |  6 ++++--
 5 files changed, 30 insertions(+), 18 deletions(-)

diff --git a/src/msgr_receive.cpp b/src/msgr_receive.cpp
index 7bff6093e..2c23a7789 100644
--- a/src/msgr_receive.cpp
+++ b/src/msgr_receive.cpp
@@ -296,7 +296,7 @@ bool osd_messenger_t::handle_reply_hdr(osd_client_t *cl)
         delete cl->read_op;
         cl->read_op = op;
         cl->read_state = CL_READ_REPLY_DATA;
-        cl->read_remaining = op->reply.hdr.retval + (op->reply.hdr.opcode == OSD_OP_SEC_READ ? op->reply.sec_rw.attr_len : op->reply.rw.bitmap_len);
+        cl->read_remaining = op->reply.hdr.retval + bmp_len;
     }
     else if (op->reply.hdr.opcode == OSD_OP_SEC_LIST && op->reply.hdr.retval > 0)
     {
diff --git a/src/msgr_send.cpp b/src/msgr_send.cpp
index 5fd0539ff..115d5edc9 100644
--- a/src/msgr_send.cpp
+++ b/src/msgr_send.cpp
@@ -47,6 +47,27 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
         cl->sent_ops[cur_op->req.hdr.id] = cur_op;
     }
     to_outbox.push_back(NULL);
+    // Bitmap
+    if (cur_op->op_type == OSD_OP_IN &&
+        cur_op->req.hdr.opcode == OSD_OP_SEC_READ &&
+        cur_op->reply.sec_rw.attr_len > 0)
+    {
+        to_send_list.push_back((iovec){
+            .iov_base = cur_op->bitmap,
+            .iov_len = cur_op->reply.sec_rw.attr_len,
+        });
+        to_outbox.push_back(NULL);
+    }
+    else if (cur_op->op_type == OSD_OP_OUT &&
+        (cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE || cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE) &&
+        cur_op->req.sec_rw.attr_len > 0)
+    {
+        to_send_list.push_back((iovec){
+            .iov_base = cur_op->bitmap,
+            .iov_len = cur_op->req.sec_rw.attr_len,
+        });
+        to_outbox.push_back(NULL);
+    }
     // Operation data
     if ((cur_op->op_type == OSD_OP_IN
         ? (cur_op->req.hdr.opcode == OSD_OP_READ ||
@@ -59,18 +80,6 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
         cur_op->req.hdr.opcode == OSD_OP_SEC_STABILIZE ||
         cur_op->req.hdr.opcode == OSD_OP_SEC_ROLLBACK)) && cur_op->iov.count > 0)
     {
-        to_outbox.push_back(NULL);
-        // Bitmap
-        if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ && cur_op->reply.sec_rw.attr_len > 0 ||
-            (cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE || cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE) &&
-            cur_op->req.sec_rw.attr_len > 0)
-        {
-            to_send_list.push_back((iovec){
-                .iov_base = cur_op->bitmap,
-                .iov_len = cur_op->reply.sec_rw.attr_len,
-            });
-            to_outbox.push_back(NULL);
-        }
         for (int i = 0; i < cur_op->iov.count; i++)
         {
             assert(cur_op->iov.buf[i].iov_base);
diff --git a/src/osd_primary.cpp b/src/osd_primary.cpp
index f16deb383..8994d695f 100644
--- a/src/osd_primary.cpp
+++ b/src/osd_primary.cpp
@@ -106,6 +106,7 @@ void osd_t::continue_primary_read(osd_op_t *cur_op)
     {
         return;
     }
+    cur_op->reply.rw.bitmap_len = 0;
     osd_primary_op_data_t *op_data = cur_op->op_data;
     if (op_data->st == 1)      goto resume_1;
     else if (op_data->st == 2) goto resume_2;
@@ -179,10 +180,10 @@ resume_2:
     }
     else
     {
-        cur_op->reply.rw.bitmap_len = op_data->pg_data_size * entry_attr_size;
-        cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
         cur_op->iov.push_back(cur_op->buf, cur_op->req.rw.len);
     }
+    cur_op->reply.rw.bitmap_len = op_data->pg_data_size * entry_attr_size;
+    cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
     finish_op(cur_op, cur_op->req.rw.len);
 }
 
diff --git a/src/osd_primary_subops.cpp b/src/osd_primary_subops.cpp
index ec8209d61..1b6e4f5bc 100644
--- a/src/osd_primary_subops.cpp
+++ b/src/osd_primary_subops.cpp
@@ -198,7 +198,7 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, int pg_s
                     .version = op_version,
                     .offset = wr ? stripes[stripe_num].write_start : stripes[stripe_num].read_start,
                     .len = wr ? stripes[stripe_num].write_end - stripes[stripe_num].write_start : stripes[stripe_num].read_end - stripes[stripe_num].read_start,
-                    .attr_len = entry_attr_size,
+                    .attr_len = wr ? entry_attr_size : 0,
                 };
 #ifdef OSD_DEBUG
                 printf(
diff --git a/src/osd_secondary.cpp b/src/osd_secondary.cpp
index bb596612a..7d9305fa6 100644
--- a/src/osd_secondary.cpp
+++ b/src/osd_secondary.cpp
@@ -19,10 +19,12 @@ void osd_t::secondary_op_callback(osd_op_t *op)
     }
     if (op->req.hdr.opcode == OSD_OP_SEC_READ)
     {
+        if (op->bs_op->retval >= 0)
+            op->reply.sec_rw.attr_len = entry_attr_size;
+        else
+            op->reply.sec_rw.attr_len = 0;
         if (op->bs_op->retval > 0)
-        {
             op->iov.push_back(op->buf, op->bs_op->retval);
-        }
     }
     else if (op->req.hdr.opcode == OSD_OP_SEC_LIST)
     {