Add a test with OSD kills
parent
2bdf415eb3
commit
82b9f4c52d
|
@ -36,6 +36,7 @@ struct sec_data
|
||||||
/* The list of completed io_u structs. */
|
/* The list of completed io_u structs. */
|
||||||
std::vector<io_u*> completed;
|
std::vector<io_u*> completed;
|
||||||
uint64_t inflight = 0;
|
uint64_t inflight = 0;
|
||||||
|
int mirror_fd = -1;
|
||||||
bool trace = false;
|
bool trace = false;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -46,6 +47,7 @@ struct sec_options
|
||||||
char *etcd_host = NULL;
|
char *etcd_host = NULL;
|
||||||
char *etcd_prefix = NULL;
|
char *etcd_prefix = NULL;
|
||||||
char *image = NULL;
|
char *image = NULL;
|
||||||
|
char *mirror_file = NULL;
|
||||||
uint64_t pool = 0;
|
uint64_t pool = 0;
|
||||||
uint64_t inode = 0;
|
uint64_t inode = 0;
|
||||||
int cluster_log = 0;
|
int cluster_log = 0;
|
||||||
|
@ -132,6 +134,15 @@ static struct fio_option options[] = {
|
||||||
.category = FIO_OPT_C_ENGINE,
|
.category = FIO_OPT_C_ENGINE,
|
||||||
.group = FIO_OPT_G_FILENAME,
|
.group = FIO_OPT_G_FILENAME,
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
.name = "mirror_file",
|
||||||
|
.lname = "File name to mirror writes to",
|
||||||
|
.type = FIO_OPT_STR_STORE,
|
||||||
|
.off1 = offsetof(struct sec_options, mirror_file),
|
||||||
|
.help = "File name to mirror writes to (for debug purpose)",
|
||||||
|
.category = FIO_OPT_C_ENGINE,
|
||||||
|
.group = FIO_OPT_G_FILENAME,
|
||||||
|
},
|
||||||
{
|
{
|
||||||
.name = "use_rdma",
|
.name = "use_rdma",
|
||||||
.lname = "Use RDMA",
|
.lname = "Use RDMA",
|
||||||
|
@ -212,6 +223,16 @@ static int sec_setup(struct thread_data *td)
|
||||||
td->o.open_files++;
|
td->o.open_files++;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (o->mirror_file)
|
||||||
|
{
|
||||||
|
bsd->mirror_fd = open(o->mirror_file, O_CREAT|O_RDWR, 0666);
|
||||||
|
if (bsd->mirror_fd < 0)
|
||||||
|
{
|
||||||
|
td_verror(td, errno, "open mirror file");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if (!o->image)
|
if (!o->image)
|
||||||
{
|
{
|
||||||
if (!(o->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)))
|
if (!(o->inode & (((uint64_t)1 << (64-POOL_ID_BITS)) - 1)))
|
||||||
|
@ -265,6 +286,10 @@ static void sec_cleanup(struct thread_data *td)
|
||||||
sec_data *bsd = (sec_data*)td->io_ops_data;
|
sec_data *bsd = (sec_data*)td->io_ops_data;
|
||||||
if (bsd)
|
if (bsd)
|
||||||
{
|
{
|
||||||
|
if (bsd->mirror_fd >= 0)
|
||||||
|
{
|
||||||
|
close(bsd->mirror_fd);
|
||||||
|
}
|
||||||
if (bsd->watch)
|
if (bsd->watch)
|
||||||
{
|
{
|
||||||
vitastor_c_close_watch(bsd->cli, bsd->watch);
|
vitastor_c_close_watch(bsd->cli, bsd->watch);
|
||||||
|
@ -325,6 +350,24 @@ static enum fio_q_status sec_queue(struct thread_data *td, struct io_u *io)
|
||||||
bsd->last_sync = false;
|
bsd->last_sync = false;
|
||||||
break;
|
break;
|
||||||
case DDIR_WRITE:
|
case DDIR_WRITE:
|
||||||
|
if (opt->mirror_file)
|
||||||
|
{
|
||||||
|
size_t done = 0;
|
||||||
|
while (done < io->xfer_buflen)
|
||||||
|
{
|
||||||
|
ssize_t r = pwrite(bsd->mirror_fd, io->xfer_buf+done, io->xfer_buflen-done, io->offset+done);
|
||||||
|
if (r < 0 && errno != EAGAIN)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Error writing mirror file: %s\n", strerror(errno));
|
||||||
|
io->error = errno;
|
||||||
|
return FIO_Q_COMPLETED;
|
||||||
|
}
|
||||||
|
if (r > 0)
|
||||||
|
{
|
||||||
|
done += r;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
if (opt->image && vitastor_c_inode_get_readonly(bsd->watch))
|
if (opt->image && vitastor_c_inode_get_readonly(bsd->watch))
|
||||||
{
|
{
|
||||||
io->error = EROFS;
|
io->error = EROFS;
|
||||||
|
|
|
@ -4,19 +4,25 @@
|
||||||
|
|
||||||
if [ "$IMMEDIATE_COMMIT" != "" ]; then
|
if [ "$IMMEDIATE_COMMIT" != "" ]; then
|
||||||
NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 1"
|
NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 1"
|
||||||
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5,"immediate_commit":"all"}'
|
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1,"immediate_commit":"all"}'
|
||||||
else
|
else
|
||||||
NO_SAME="--journal_sector_buffer_count 1024 --log_level 1"
|
NO_SAME="--journal_sector_buffer_count 1024 --log_level 1"
|
||||||
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5}'
|
$ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1}'
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
start_osd()
|
||||||
|
{
|
||||||
|
local i=$1
|
||||||
|
build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
|
||||||
|
eval OSD${i}_PID=$!
|
||||||
|
}
|
||||||
|
|
||||||
OSD_SIZE=1024
|
OSD_SIZE=1024
|
||||||
OSD_COUNT=7
|
OSD_COUNT=7
|
||||||
OSD_ARGS=
|
OSD_ARGS=
|
||||||
for i in $(seq 1 $OSD_COUNT); do
|
for i in $(seq 1 $OSD_COUNT); do
|
||||||
dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
|
dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
|
||||||
build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
|
start_osd $i
|
||||||
eval OSD${i}_PID=$!
|
|
||||||
done
|
done
|
||||||
|
|
||||||
cd mon
|
cd mon
|
||||||
|
@ -26,11 +32,11 @@ node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1
|
||||||
MON_PID=$!
|
MON_PID=$!
|
||||||
|
|
||||||
if [ "$EC" != "" ]; then
|
if [ "$EC" != "" ]; then
|
||||||
POOLCFG='"scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1'
|
|
||||||
PG_SIZE=3
|
PG_SIZE=3
|
||||||
|
POOLCFG='"scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1'
|
||||||
else
|
else
|
||||||
POOLCFG='"scheme":"replicated","pg_size":2,"pg_minsize":2'
|
PG_SIZE=${PG_SIZE:-2}
|
||||||
PG_SIZE=2
|
POOLCFG='"scheme":"replicated","pg_size":'$PG_SIZE',"pg_minsize":2'
|
||||||
fi
|
fi
|
||||||
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":32,"failure_domain":"osd"}}'
|
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":32,"failure_domain":"osd"}}'
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,52 @@
|
||||||
|
#!/bin/bash -ex
|
||||||
|
|
||||||
|
# Kill OSDs while writing
|
||||||
|
|
||||||
|
PG_SIZE=3
|
||||||
|
|
||||||
|
. `dirname $0`/run_7osds.sh
|
||||||
|
|
||||||
|
IMG_SIZE=960
|
||||||
|
|
||||||
|
$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
|
||||||
|
|
||||||
|
LD_PRELOAD="build/src/libfio_vitastor.so" \
|
||||||
|
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \
|
||||||
|
-mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -cluster_log_level=10
|
||||||
|
|
||||||
|
kill_osds()
|
||||||
|
{
|
||||||
|
sleep 5
|
||||||
|
|
||||||
|
kill -9 $OSD1_PID
|
||||||
|
$ETCDCTL del /vitastor/osd/state/1
|
||||||
|
|
||||||
|
for i in 2 3 4 5 6 7; do
|
||||||
|
sleep 15
|
||||||
|
echo Killing OSD $i and starting OSD $((i-1))
|
||||||
|
p=OSD${i}_PID
|
||||||
|
kill -9 ${!p}
|
||||||
|
$ETCDCTL del /vitastor/osd/state/$i
|
||||||
|
start_osd $((i-1))
|
||||||
|
sleep 15
|
||||||
|
done
|
||||||
|
|
||||||
|
sleep 5
|
||||||
|
start_osd 7
|
||||||
|
|
||||||
|
sleep 5
|
||||||
|
}
|
||||||
|
|
||||||
|
kill_osds &
|
||||||
|
|
||||||
|
LD_PRELOAD="build/src/libfio_vitastor.so" \
|
||||||
|
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=16 -fsync=256 -rw=randwrite \
|
||||||
|
-mirror_file=./testdata/mirror.bin -etcd=$ETCD_URL -image=testimg -loops=10 -runtime=120 2>/dev/null
|
||||||
|
|
||||||
|
qemu-img convert -S 4096 -p \
|
||||||
|
-f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:image=testimg" \
|
||||||
|
-O raw ./testdata/read.bin
|
||||||
|
|
||||||
|
diff ./testdata/read.bin ./testdata/mirror.bin
|
||||||
|
|
||||||
|
format_green OK
|
Loading…
Reference in New Issue