vmsplice+splice experiment in stub_osd to test it too

src/CMakeLists.txt

@@ -188,7 +188,7 @@ endif (${WITH_QEMU})
 ### Test stubs
 
 # stub_osd, stub_bench, osd_test
-add_executable(stub_osd stub_osd.cpp rw_blocking.cpp)
+add_executable(stub_osd stub_osd.cpp rw_blocking.cpp mmap_manager.cpp)
 target_link_libraries(stub_osd tcmalloc_minimal)
 add_executable(stub_bench stub_bench.cpp rw_blocking.cpp)
 target_link_libraries(stub_bench tcmalloc_minimal)

src/stub_osd.cpp

@@ -37,6 +37,7 @@
 
 #include <stdexcept>
 
+#include "mmap_manager.h"
 #include "rw_blocking.h"
 #include "osd_ops.h"
 
@@ -115,6 +116,12 @@ int bind_stub(const char *bind_address, int bind_port)
 
 void run_stub(int peer_fd)
 {
+    mmap_manager_t mm;
+    int pipe_fd[2];
+    if (pipe(pipe_fd) < 0)
+    {
+        throw std::runtime_error(std::string("pipe: ") + strerror(errno));
+    }
     osd_any_op_t op;
     osd_any_reply_t reply;
     void *buf = NULL;
@@ -136,11 +143,39 @@ void run_stub(int peer_fd)
         if (op.hdr.opcode == OSD_OP_SEC_READ)
         {
             reply.hdr.retval = op.sec_rw.len;
-            buf = malloc(op.sec_rw.len);
+            //buf = malloc(op.sec_rw.len);
+            buf = mm.alloc(op.sec_rw.len);
             r = write_blocking(peer_fd, reply.buf, OSD_PACKET_SIZE);
             if (r == OSD_PACKET_SIZE)
-                r = write_blocking(peer_fd, &buf, op.sec_rw.len);
-            free(buf);
+            {
+                size_t offset = 0;
+                while (offset < op.sec_rw.len)
+                {
+                    iovec iov = { .iov_base = buf+offset, .iov_len = op.sec_rw.len-offset };
+                    int vmspliced = vmsplice(pipe_fd[1], &iov, 1, SPLICE_F_GIFT);
+                    if (vmspliced < 0)
+                    {
+                        throw std::runtime_error(std::string("vmsplice: ")+strerror(errno));
+                    }
+                    int spliced = 0;
+                    while (spliced < vmspliced)
+                    {
+                        int r2 = splice(pipe_fd[0], NULL, peer_fd, NULL, vmspliced-spliced, SPLICE_F_MOVE);
+                        if (r2 < 0)
+                        {
+                            if (errno != EAGAIN)
+                                throw std::runtime_error(std::string("splice: ")+strerror(errno));
+                        }
+                        else
+                            spliced += r2;
+                    }
+                    offset += vmspliced;
+                }
+                r = offset;
+                //r = write_blocking(peer_fd, &buf, op.sec_rw.len);
+            }
+            mm.free(buf, op.sec_rw.len);
+            buf = NULL;
             if (r < op.sec_rw.len)
                 break;
         }
@@ -170,5 +205,6 @@ void run_stub(int peer_fd)
             break;
         }
     }
-    free(buf);
+    close(pipe_fd[0]);
+    close(pipe_fd[1]);
 }