Add a test with OSD kills

rm-left-on-dead
Vitaliy Filippov 2022-05-21 10:08:08 +03:00
parent 2bdf415eb3
commit 82b9f4c52d
3 changed files with 108 additions and 7 deletions

View File

@ -36,6 +36,7 @@ struct sec_data
/* The list of completed io_u structs. */
std::vector<io_u*> completed;
uint64_t inflight = 0;
int mirror_fd = -1;
bool trace = false;
};
@ -46,6 +47,7 @@ struct sec_options
char *etcd_host = NULL;
char *etcd_prefix = NULL;
char *image = NULL;
char *mirror_file = NULL;
uint64_t pool = 0;
uint64_t inode = 0;
int cluster_log = 0;
@ -132,6 +134,15 @@ static struct fio_option options[] = {
.category = FIO_OPT_C_ENGINE,
.group = FIO_OPT_G_FILENAME,
},
{
.name = "mirror_file",
.lname = "File name to mirror writes to",
.type = FIO_OPT_STR_STORE,
.off1 = offsetof(struct sec_options, mirror_file),
.help = "File name to mirror writes to (for debug purpose)",
.category = FIO_OPT_C_ENGINE,
.group = FIO_OPT_G_FILENAME,
},
{
.name = "use_rdma",
.lname = "Use RDMA",
@ -212,6 +223,16 @@ static int sec_setup(struct thread_data *td)
td->o.open_files++;
}
if (o->mirror_file)
{
bsd->mirror_fd = open(o->mirror_file, O_CREAT|O_RDWR, 0666);
if (bsd->mirror_fd < 0)
{
td_verror(td, errno, "open mirror file");
return 1;
}
}
if (!o->image)
{
if (!(o->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)))
@ -265,6 +286,10 @@ static void sec_cleanup(struct thread_data *td)
sec_data *bsd = (sec_data*)td->io_ops_data;
if (bsd)
{
if (bsd->mirror_fd >= 0)
{
close(bsd->mirror_fd);
}
if (bsd->watch)
{
vitastor_c_close_watch(bsd->cli, bsd->watch);
@ -325,6 +350,24 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
bsd->last_sync = false;
break;
case DDIR_WRITE:
if (opt->mirror_file)
{
size_t done = 0;
while (done < io->xfer_buflen)
{
ssize_t r = pwrite(bsd->mirror_fd, io->xfer_buf+done, io->xfer_buflen-done, io->offset+done);
if (r < 0 && errno != EAGAIN)
{
fprintf(stderr, "Error writing mirror file: %s\n", strerror(errno));
io->error = errno;
return FIO_Q_COMPLETED;
}
if (r > 0)
{
done += r;
}
}
}
if (opt->image && vitastor_c_inode_get_readonly(bsd->watch))
{
io->error = EROFS;

View File

@ -4,19 +4,25 @@
if [ "$IMMEDIATE_COMMIT" != "" ]; then
NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 1"
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5,"immediate_commit":"all"}'
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1,"immediate_commit":"all"}'
else
NO_SAME="--journal_sector_buffer_count 1024 --log_level 1"
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5}'
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1}'
fi
start_osd()
{
local i=$1
build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
eval OSD${i}_PID=$!
}
OSD_SIZE=1024
OSD_COUNT=7
OSD_ARGS=
for i in $(seq 1 $OSD_COUNT); do
dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
eval OSD${i}_PID=$!
start_osd $i
done
cd mon
@ -26,11 +32,11 @@ node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1
MON_PID=$!
if [ "$EC" != "" ]; then
POOLCFG='"scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1'
PG_SIZE=3
POOLCFG='"scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1'
else
POOLCFG='"scheme":"replicated","pg_size":2,"pg_minsize":2'
PG_SIZE=2
PG_SIZE=${PG_SIZE:-2}
POOLCFG='"scheme":"replicated","pg_size":'$PG_SIZE',"pg_minsize":2'
fi
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":32,"failure_domain":"osd"}}'

52
tests/test_heal.sh Executable file
View File

@ -0,0 +1,52 @@
#!/bin/bash -ex
# Kill OSDs while writing
PG_SIZE=3
. `dirname $0`/run_7osds.sh
IMG_SIZE=960
$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
LD_PRELOAD="build/src/libfio_vitastor.so" \
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
-mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -cluster_log_level=10
kill_osds()
{
sleep 5
kill -9 $OSD1_PID
$ETCDCTL del /vitastor/osd/state/1
for i in 2 3 4 5 6 7; do
sleep 15
echo Killing OSD $i and starting OSD $((i-1))
p=OSD${i}_PID
kill -9 ${!p}
$ETCDCTL del /vitastor/osd/state/$i
start_osd $((i-1))
sleep 15
done
sleep 5
start_osd 7
sleep 5
}
kill_osds &
LD_PRELOAD="build/src/libfio_vitastor.so" \
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=16 -fsync=256 -rw=randwrite \
-mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=10 -runtime=120 2>/dev/null
qemu-img convert -S 4096 -p \
-f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \
-O raw ./testdata/read.bin
diff ./testdata/read.bin ./testdata/mirror.bin
format_green OK