Add a test with OSD kills

src/fio_cluster.cpp

@@ -36,6 +36,7 @@ struct sec_data
     /* The list of completed io_u structs. */
     std::vector<io_u*> completed;
     uint64_t inflight = 0;
+    int mirror_fd = -1;
     bool trace = false;
 };
 
@@ -46,6 +47,7 @@ struct sec_options
     char *etcd_host = NULL;
     char *etcd_prefix = NULL;
     char *image = NULL;
+    char *mirror_file = NULL;
     uint64_t pool = 0;
     uint64_t inode = 0;
     int cluster_log = 0;
@@ -132,6 +134,15 @@ static struct fio_option options[] = {
         .category = FIO_OPT_C_ENGINE,
         .group  = FIO_OPT_G_FILENAME,
     },
+    {
+        .name   = "mirror_file",
+        .lname  = "File name to mirror writes to",
+        .type   = FIO_OPT_STR_STORE,
+        .off1   = offsetof(struct sec_options, mirror_file),
+        .help   = "File name to mirror writes to (for debug purpose)",
+        .category = FIO_OPT_C_ENGINE,
+        .group  = FIO_OPT_G_FILENAME,
+    },
     {
         .name   = "use_rdma",
         .lname  = "Use RDMA",
@@ -212,6 +223,16 @@ static int sec_setup(struct thread_data *td)
         td->o.open_files++;
     }
 
+    if (o->mirror_file)
+    {
+        bsd->mirror_fd = open(o->mirror_file, O_CREAT|O_RDWR, 0666);
+        if (bsd->mirror_fd < 0)
+        {
+            td_verror(td, errno, "open mirror file");
+            return 1;
+        }
+    }
+
     if (!o->image)
     {
         if (!(o->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)))
@@ -265,6 +286,10 @@ static void sec_cleanup(struct thread_data *td)
     sec_data *bsd = (sec_data*)td->io_ops_data;
     if (bsd)
     {
+        if (bsd->mirror_fd >= 0)
+        {
+            close(bsd->mirror_fd);
+        }
         if (bsd->watch)
         {
             vitastor_c_close_watch(bsd->cli, bsd->watch);
@@ -325,6 +350,24 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
         bsd->last_sync = false;
         break;
     case DDIR_WRITE:
+        if (opt->mirror_file)
+        {
+            size_t done = 0;
+            while (done < io->xfer_buflen)
+            {
+                ssize_t r = pwrite(bsd->mirror_fd, io->xfer_buf+done, io->xfer_buflen-done, io->offset+done);
+                if (r < 0 && errno != EAGAIN)
+                {
+                    fprintf(stderr, "Error writing mirror file: %s\n", strerror(errno));
+                    io->error = errno;
+                    return FIO_Q_COMPLETED;
+                }
+                if (r > 0)
+                {
+                    done += r;
+                }
+            }
+        }
         if (opt->image && vitastor_c_inode_get_readonly(bsd->watch))
         {
             io->error = EROFS;

tests/run_7osds.sh

@@ -4,19 +4,25 @@
 
 if [ "$IMMEDIATE_COMMIT" != "" ]; then
     NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 1"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5,"immediate_commit":"all"}'
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1,"immediate_commit":"all"}'
 else
     NO_SAME="--journal_sector_buffer_count 1024 --log_level 1"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5}'
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1}'
 fi
 
+start_osd()
+{
+    local i=$1
+    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
+    eval OSD${i}_PID=$!
+}
+
 OSD_SIZE=1024
 OSD_COUNT=7
 OSD_ARGS=
 for i in $(seq 1 $OSD_COUNT); do
     dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
-    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
-    eval OSD${i}_PID=$!
+    start_osd $i
 done
 
 cd mon
@@ -26,11 +32,11 @@ node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1
 MON_PID=$!
 
 if [ "$EC" != "" ]; then
-    POOLCFG='"scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1'
     PG_SIZE=3
+    POOLCFG='"scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1'
 else
-    POOLCFG='"scheme":"replicated","pg_size":2,"pg_minsize":2'
-    PG_SIZE=2
+    PG_SIZE=${PG_SIZE:-2}
+    POOLCFG='"scheme":"replicated","pg_size":'$PG_SIZE',"pg_minsize":2'
 fi
 $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":32,"failure_domain":"osd"}}'
 

tests/test_heal.sh

@@ -0,0 +1,52 @@
+#!/bin/bash -ex
+
+# Kill OSDs while writing
+
+PG_SIZE=3
+
+. `dirname $0`/run_7osds.sh
+
+IMG_SIZE=960
+
+$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
+
+LD_PRELOAD="build/src/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
+        -mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -cluster_log_level=10
+
+kill_osds()
+{
+    sleep 5
+
+    kill -9 $OSD1_PID
+    $ETCDCTL del /vitastor/osd/state/1
+
+    for i in 2 3 4 5 6 7; do
+        sleep 15
+        echo Killing OSD $i and starting OSD $((i-1))
+        p=OSD${i}_PID
+        kill -9 ${!p}
+        $ETCDCTL del /vitastor/osd/state/$i
+        start_osd $((i-1))
+        sleep 15
+    done
+
+    sleep 5
+    start_osd 7
+
+    sleep 5
+}
+
+kill_osds &
+
+LD_PRELOAD="build/src/libfio_vitastor.so" \
+    fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=16 -fsync=256 -rw=randwrite \
+        -mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=10 -runtime=120 2>/dev/null
+
+qemu-img convert -S 4096 -p \
+    -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \
+    -O raw ./testdata/read.bin
+
+diff ./testdata/read.bin ./testdata/mirror.bin
+
+format_green OK