Fix several snapshot I/O bugs

src/msgr_receive.cpp

@@ -296,7 +296,7 @@ bool osd_messenger_t::handle_reply_hdr(osd_client_t *cl)
         delete cl->read_op;
         cl->read_op = op;
         cl->read_state = CL_READ_REPLY_DATA;
-        cl->read_remaining = op->reply.hdr.retval + (op->reply.hdr.opcode == OSD_OP_SEC_READ ? op->reply.sec_rw.attr_len : op->reply.rw.bitmap_len);
+        cl->read_remaining = op->reply.hdr.retval + bmp_len;
     }
     else if (op->reply.hdr.opcode == OSD_OP_SEC_LIST && op->reply.hdr.retval > 0)
     {

src/msgr_send.cpp

@@ -47,6 +47,27 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
         cl->sent_ops[cur_op->req.hdr.id] = cur_op;
     }
     to_outbox.push_back(NULL);
+    // Bitmap
+    if (cur_op->op_type == OSD_OP_IN &&
+        cur_op->req.hdr.opcode == OSD_OP_SEC_READ &&
+        cur_op->reply.sec_rw.attr_len > 0)
+    {
+        to_send_list.push_back((iovec){
+            .iov_base = cur_op->bitmap,
+            .iov_len = cur_op->reply.sec_rw.attr_len,
+        });
+        to_outbox.push_back(NULL);
+    }
+    else if (cur_op->op_type == OSD_OP_OUT &&
+        (cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE || cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE) &&
+        cur_op->req.sec_rw.attr_len > 0)
+    {
+        to_send_list.push_back((iovec){
+            .iov_base = cur_op->bitmap,
+            .iov_len = cur_op->req.sec_rw.attr_len,
+        });
+        to_outbox.push_back(NULL);
+    }
     // Operation data
     if ((cur_op->op_type == OSD_OP_IN
         ? (cur_op->req.hdr.opcode == OSD_OP_READ ||
@@ -59,18 +80,6 @@ void osd_messenger_t::outbox_push(osd_op_t *cur_op)
         cur_op->req.hdr.opcode == OSD_OP_SEC_STABILIZE ||
         cur_op->req.hdr.opcode == OSD_OP_SEC_ROLLBACK)) && cur_op->iov.count > 0)
     {
-        to_outbox.push_back(NULL);
-        // Bitmap
-        if (cur_op->req.hdr.opcode == OSD_OP_SEC_READ && cur_op->reply.sec_rw.attr_len > 0 ||
-            (cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE || cur_op->req.hdr.opcode == OSD_OP_SEC_WRITE_STABLE) &&
-            cur_op->req.sec_rw.attr_len > 0)
-        {
-            to_send_list.push_back((iovec){
-                .iov_base = cur_op->bitmap,
-                .iov_len = cur_op->reply.sec_rw.attr_len,
-            });
-            to_outbox.push_back(NULL);
-        }
         for (int i = 0; i < cur_op->iov.count; i++)
         {
             assert(cur_op->iov.buf[i].iov_base);

src/osd_primary.cpp

@@ -106,6 +106,7 @@ void osd_t::continue_primary_read(osd_op_t *cur_op)
     {
         return;
     }
+    cur_op->reply.rw.bitmap_len = 0;
     osd_primary_op_data_t *op_data = cur_op->op_data;
     if (op_data->st == 1)      goto resume_1;
     else if (op_data->st == 2) goto resume_2;
@@ -179,10 +180,10 @@ resume_2:
     }
     else
     {
-        cur_op->reply.rw.bitmap_len = op_data->pg_data_size * entry_attr_size;
-        cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
         cur_op->iov.push_back(cur_op->buf, cur_op->req.rw.len);
     }
+    cur_op->reply.rw.bitmap_len = op_data->pg_data_size * entry_attr_size;
+    cur_op->iov.push_back(op_data->stripes[0].bmp_buf, cur_op->reply.rw.bitmap_len);
     finish_op(cur_op, cur_op->req.rw.len);
 }
 

src/osd_primary_subops.cpp

@@ -198,7 +198,7 @@ void osd_t::submit_primary_subops(int submit_type, uint64_t op_version, int pg_s
                     .version = op_version,
                     .offset = wr ? stripes[stripe_num].write_start : stripes[stripe_num].read_start,
                     .len = wr ? stripes[stripe_num].write_end - stripes[stripe_num].write_start : stripes[stripe_num].read_end - stripes[stripe_num].read_start,
-                    .attr_len = entry_attr_size,
+                    .attr_len = wr ? entry_attr_size : 0,
                 };
 #ifdef OSD_DEBUG
                 printf(

src/osd_secondary.cpp

@@ -19,10 +19,12 @@ void osd_t::secondary_op_callback(osd_op_t *op)
     }
     if (op->req.hdr.opcode == OSD_OP_SEC_READ)
     {
+        if (op->bs_op->retval >= 0)
+            op->reply.sec_rw.attr_len = entry_attr_size;
+        else
+            op->reply.sec_rw.attr_len = 0;
         if (op->bs_op->retval > 0)
-        {
             op->iov.push_back(op->buf, op->bs_op->retval);
-        }
     }
     else if (op->req.hdr.opcode == OSD_OP_SEC_LIST)
     {