diff --git a/src/fio_cluster.cpp b/src/fio_cluster.cpp index 438cadb8..1a3195da 100644 --- a/src/fio_cluster.cpp +++ b/src/fio_cluster.cpp @@ -36,6 +36,7 @@ struct sec_data /* The list of completed io_u structs. */ std::vector completed; uint64_t inflight = 0; + int mirror_fd = -1; bool trace = false; }; @@ -46,6 +47,7 @@ struct sec_options char *etcd_host = NULL; char *etcd_prefix = NULL; char *image = NULL; + char *mirror_file = NULL; uint64_t pool = 0; uint64_t inode = 0; int cluster_log = 0; @@ -132,6 +134,15 @@ static struct fio_option options[] = { .category = FIO_OPT_C_ENGINE, .group = FIO_OPT_G_FILENAME, }, + { + .name = "mirror_file", + .lname = "File name to mirror writes to", + .type = FIO_OPT_STR_STORE, + .off1 = offsetof(struct sec_options, mirror_file), + .help = "File name to mirror writes to (for debug purpose)", + .category = FIO_OPT_C_ENGINE, + .group = FIO_OPT_G_FILENAME, + }, { .name = "use_rdma", .lname = "Use RDMA", @@ -212,6 +223,16 @@ static int sec_setup(struct thread_data *td) td->o.open_files++; } + if (o->mirror_file) + { + bsd->mirror_fd = open(o->mirror_file, O_CREAT|O_RDWR, 0666); + if (bsd->mirror_fd < 0) + { + td_verror(td, errno, "open mirror file"); + return 1; + } + } + if (!o->image) { if (!(o->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1))) @@ -265,6 +286,10 @@ static void sec_cleanup(struct thread_data *td) sec_data *bsd = (sec_data*)td->io_ops_data; if (bsd) { + if (bsd->mirror_fd >= 0) + { + close(bsd->mirror_fd); + } if (bsd->watch) { vitastor_c_close_watch(bsd->cli, bsd->watch); @@ -325,6 +350,24 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io) bsd->last_sync = false; break; case DDIR_WRITE: + if (opt->mirror_file) + { + size_t done = 0; + while (done < io->xfer_buflen) + { + ssize_t r = pwrite(bsd->mirror_fd, io->xfer_buf+done, io->xfer_buflen-done, io->offset+done); + if (r < 0 && errno != EAGAIN) + { + fprintf(stderr, "Error writing mirror file: %s\n", strerror(errno)); + io->error = errno; + return FIO_Q_COMPLETED; + } + if (r > 0) + { + done += r; + } + } + } if (opt->image && vitastor_c_inode_get_readonly(bsd->watch)) { io->error = EROFS; diff --git a/tests/run_7osds.sh b/tests/run_7osds.sh index 41ae2f7a..def9e035 100644 --- a/tests/run_7osds.sh +++ b/tests/run_7osds.sh @@ -4,19 +4,25 @@ if [ "$IMMEDIATE_COMMIT" != "" ]; then NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 1" - $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5,"immediate_commit":"all"}' + $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1,"immediate_commit":"all"}' else NO_SAME="--journal_sector_buffer_count 1024 --log_level 1" - $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5}' + $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1}' fi +start_osd() +{ + local i=$1 + build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log & + eval OSD${i}_PID=$! +} + OSD_SIZE=1024 OSD_COUNT=7 OSD_ARGS= for i in $(seq 1 $OSD_COUNT); do dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1)) - build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log & - eval OSD${i}_PID=$! + start_osd $i done cd mon @@ -26,11 +32,11 @@ node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 MON_PID=$! if [ "$EC" != "" ]; then - POOLCFG='"scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1' PG_SIZE=3 + POOLCFG='"scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1' else - POOLCFG='"scheme":"replicated","pg_size":2,"pg_minsize":2' - PG_SIZE=2 + PG_SIZE=${PG_SIZE:-2} + POOLCFG='"scheme":"replicated","pg_size":'$PG_SIZE',"pg_minsize":2' fi $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":32,"failure_domain":"osd"}}' diff --git a/tests/test_heal.sh b/tests/test_heal.sh new file mode 100755 index 00000000..fb3598da --- /dev/null +++ b/tests/test_heal.sh @@ -0,0 +1,52 @@ +#!/bin/bash -ex + +# Kill OSDs while writing + +PG_SIZE=3 + +. `dirname $0`/run_7osds.sh + +IMG_SIZE=960 + +$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}' + +LD_PRELOAD="build/src/libfio_vitastor.so" \ + fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \ + -mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -cluster_log_level=10 + +kill_osds() +{ + sleep 5 + + kill -9 $OSD1_PID + $ETCDCTL del /vitastor/osd/state/1 + + for i in 2 3 4 5 6 7; do + sleep 15 + echo Killing OSD $i and starting OSD $((i-1)) + p=OSD${i}_PID + kill -9 ${!p} + $ETCDCTL del /vitastor/osd/state/$i + start_osd $((i-1)) + sleep 15 + done + + sleep 5 + start_osd 7 + + sleep 5 +} + +kill_osds & + +LD_PRELOAD="build/src/libfio_vitastor.so" \ + fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=16 -fsync=256 -rw=randwrite \ + -mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=10 -runtime=120 2>/dev/null + +qemu-img convert -S 4096 -p \ + -f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \ + -O raw ./testdata/read.bin + +diff ./testdata/read.bin ./testdata/mirror.bin + +format_green OK