diff --git a/tests/run_3osds.sh b/tests/run_3osds.sh index a507a98d..898a9f5e 100644 --- a/tests/run_3osds.sh +++ b/tests/run_3osds.sh @@ -1,47 +1,103 @@ -#!/bin/bash -ex +#!/bin/bash . `dirname $0`/common.sh OSD_SIZE=${OSD_SIZE:-1024} PG_COUNT=${PG_COUNT:-1} -PG_SIZE=${PG_SIZE:-3} -PG_MINSIZE=${PG_MINSIZE:-2} -OSD_COUNT=${OSD_COUNT:-3} -SCHEME=${SCHEME:-ec} +# OSD_COUNT +SCHEME=${SCHEME:-replicated} +# OSD_ARGS +# PG_SIZE +# PG_MINSIZE + +if [ "$SCHEME" = "ec" ]; then + OSD_COUNT=${OSD_COUNT:-5} +else + OSD_COUNT=${OSD_COUNT:-3} +fi + +if [ "$IMMEDIATE_COMMIT" != "" ]; then + NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 1" + $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1,"immediate_commit":"all"}' +else + NO_SAME="--journal_sector_buffer_count 1024 --log_level 1" + $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1}' +fi + +start_osd() +{ + local i=$1 + build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log & + eval OSD${i}_PID=$! +} for i in $(seq 1 $OSD_COUNT); do dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1)) - build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log & - eval OSD${i}_PID=$! + start_osd $i done cd mon npm install cd .. -node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & +node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log & MON_PID=$! -if [ -n "$GLOBAL_CONF" ]; then - $ETCDCTL put /vitastor/config/global "$GLOBAL_CONF" -fi - -if [ "$SCHEME" = "replicated" ]; then - $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$PG_COUNT',"failure_domain":"osd"}}' +if [ "$SCHEME" = "ec" ]; then + PG_SIZE=${PG_SIZE:-5} + PG_MINSIZE=${PG_MINSIZE:-3} + PG_DATA_SIZE=$PG_MINSIZE + POOLCFG='"scheme":"jerasure","parity_chunks":'$((PG_SIZE-PG_MINSIZE)) +elif [ "$SCHEME" = "xor" ]; then + PG_SIZE=${PG_SIZE:-3} + PG_MINSIZE=${PG_MINSIZE:-2} + PG_DATA_SIZE=$PG_MINSIZE + POOLCFG='"scheme":"xor","parity_chunks":'$((PG_SIZE-PG_MINSIZE)) else - $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"parity_chunks":1,"pg_count":'$PG_COUNT',"failure_domain":"osd"}}' + PG_SIZE=${PG_SIZE:-2} + PG_MINSIZE=${PG_MINSIZE:-2} + PG_DATA_SIZE=1 + POOLCFG='"scheme":"replicated"' +fi +POOLCFG='"name":"testpool","failure_domain":"osd",'$POOLCFG +$ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$PG_COUNT'}}' + +sleep 2 + +if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == '$PG_SIZE') | length) == '$PG_COUNT); then + format_error "FAILED: $PG_COUNT PGS NOT CONFIGURED" fi -sleep 3 - -if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and ([ .[0].items["1"][] | select(((.osd_set | select(. != 0) | sort | unique) | length) == '$PG_SIZE') ] | length) == '$PG_COUNT); then - format_error "FAILED: $PG_COUNT PG(s) NOT CONFIGURED" +if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == '$PG_COUNT); then + format_error "FAILED: $PG_COUNT PGS NOT UP" fi -if ! ($ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active"]) ] | length == '$PG_COUNT); then - format_error "FAILED: $PG_COUNT PG(s) NOT UP" -fi +try_reweight() +{ + osd=$1 + w=$2 + $ETCDCTL put /vitastor/config/osd/$osd '{"reweight":'$w'}' + sleep 3 +} -if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so; then - sudo rm -f /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so - sudo ln -s "$(realpath .)/build/src/block-vitastor.so" /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so -fi +wait_finish_rebalance() +{ + sec=$1 + i=0 + while [[ $i -lt $sec ]]; do + ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == 32') && \ + break + if [ $i -eq 60 ]; then + format_error "Rebalance couldn't finish in $sec seconds" + fi + sleep 1 + i=$((i+1)) + done +} + +check_qemu() +{ + if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so; then + sudo rm -f /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so + sudo ln -s "$(realpath .)/build/src/block-vitastor.so" /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so + fi +} diff --git a/tests/run_7osds.sh b/tests/run_7osds.sh deleted file mode 100644 index def9e035..00000000 --- a/tests/run_7osds.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -. `dirname $0`/common.sh - -if [ "$IMMEDIATE_COMMIT" != "" ]; then - NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 1" - $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1,"immediate_commit":"all"}' -else - NO_SAME="--journal_sector_buffer_count 1024 --log_level 1" - $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":1}' -fi - -start_osd() -{ - local i=$1 - build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log & - eval OSD${i}_PID=$! -} - -OSD_SIZE=1024 -OSD_COUNT=7 -OSD_ARGS= -for i in $(seq 1 $OSD_COUNT); do - dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1)) - start_osd $i -done - -cd mon -npm install -cd .. -node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log & -MON_PID=$! - -if [ "$EC" != "" ]; then - PG_SIZE=3 - POOLCFG='"scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1' -else - PG_SIZE=${PG_SIZE:-2} - POOLCFG='"scheme":"replicated","pg_size":'$PG_SIZE',"pg_minsize":2' -fi -$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":32,"failure_domain":"osd"}}' - -sleep 2 - -if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == '$PG_SIZE') | length) == 32'); then - format_error "FAILED: 32 PGS NOT CONFIGURED" -fi - -if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == 32'); then - format_error "FAILED: 32 PGS NOT UP" -fi - -try_reweight() -{ - osd=$1 - w=$2 - $ETCDCTL put /vitastor/config/osd/$osd '{"reweight":'$w'}' - sleep 3 -} - -wait_finish_rebalance() -{ - sec=$1 - i=0 - while [[ $i -lt $sec ]]; do - ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == 32') && \ - break - if [ $i -eq 60 ]; then - format_error "Rebalance couldn't finish in $sec seconds" - fi - sleep 1 - i=$((i+1)) - done -} diff --git a/tests/run_tests.sh b/tests/run_tests.sh index 78758d15..151e7faf 100755 --- a/tests/run_tests.sh +++ b/tests/run_tests.sh @@ -8,7 +8,7 @@ cd $(dirname $0) ./test_cas.sh ./test_change_pg_count.sh -EC=1 ./test_change_pg_count.sh +SCHEME=ec ./test_change_pg_count.sh ./test_change_pg_size.sh @@ -18,8 +18,8 @@ EC=1 ./test_change_pg_count.sh ./test_interrupted_rebalance.sh IMMEDIATE_COMMIT=1 ./test_interrupted_rebalance.sh -EC=1 ./test_interrupted_rebalance.sh -EC=1 IMMEDIATE_COMMIT=1 ./test_interrupted_rebalance.sh +SCHEME=ec ./test_interrupted_rebalance.sh +SCHEME=ec IMMEDIATE_COMMIT=1 ./test_interrupted_rebalance.sh ./test_minsize_1.sh @@ -27,17 +27,17 @@ EC=1 IMMEDIATE_COMMIT=1 ./test_interrupted_rebalance.sh ./test_rebalance_verify.sh IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh -EC=1 ./test_rebalance_verify.sh -EC=1 IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh +SCHEME=ec ./test_rebalance_verify.sh +SCHEME=ec IMMEDIATE_COMMIT=1 ./test_rebalance_verify.sh ./test_rm.sh ./test_snapshot.sh -SCHEME=replicated ./test_snapshot.sh +SCHEME=ec ./test_snapshot.sh ./test_splitbrain.sh ./test_write.sh -SCHEME=replicated ./test_write.sh +SCHEME=xor ./test_write.sh ./test_write_no_same.sh diff --git a/tests/test_change_pg_count.sh b/tests/test_change_pg_count.sh index 5b65db0b..d695b118 100755 --- a/tests/test_change_pg_count.sh +++ b/tests/test_change_pg_count.sh @@ -1,41 +1,11 @@ #!/bin/bash -ex -. `dirname $0`/common.sh +OSD_COUNT=${OSD_COUNT:-6} +PG_COUNT=16 -if [ "$EC" != "" ]; then - POOLCFG='"scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1' - NOBJ=512 -else - POOLCFG='"scheme":"replicated","pg_size":2,"pg_minsize":2' - NOBJ=1024 -fi +. `dirname $0`/run_3osds.sh -OSD_SIZE=1024 -OSD_COUNT=6 -OSD_ARGS= -for i in $(seq 1 $OSD_COUNT); do - dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1)) - build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log & - eval OSD${i}_PID=$! -done - -cd mon -npm install -cd .. -node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log & -MON_PID=$! - -$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":16,"failure_domain":"osd"}}' - -sleep 2 - -if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == 16'); then - format_error "FAILED: 16 PGS NOT CONFIGURED" -fi - -if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == 16'); then - format_error "FAILED: 16 PGS NOT UP" -fi +NOBJ=$((128*8/PG_DATA_SIZE)) LD_PRELOAD="build/src/libfio_vitastor.so" \ fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=write \ @@ -49,7 +19,7 @@ try_change() echo --- Change PG count to $n --- >>testdata/osd$i.log done - $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":'$n',"failure_domain":"osd"}}' + $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$PG_SIZE',"pg_minsize":'$PG_MINSIZE',"pg_count":'$n'}}' for i in {1..10}; do ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == '$n) && \ diff --git a/tests/test_change_pg_size.sh b/tests/test_change_pg_size.sh index a4f10dce..c2ad712b 100755 --- a/tests/test_change_pg_size.sh +++ b/tests/test_change_pg_size.sh @@ -1,40 +1,16 @@ #!/bin/bash -ex -. `dirname $0`/common.sh +PG_COUNT=16 +SCHEME=${SCHEME:-replicated} -OSD_SIZE=1024 -OSD_COUNT=3 -OSD_ARGS= -for i in $(seq 1 $OSD_COUNT); do - dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1)) - build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log & - eval OSD${i}_PID=$! -done - -cd mon -npm install -cd .. -node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & -MON_PID=$! - -$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":3,"pg_minsize":2,"pg_count":16,"failure_domain":"osd"}}' - -sleep 2 - -if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | sort) == ["1","2","3"]) | length) == 16'); then - format_error "FAILED: 16 PGS NOT CONFIGURED" -fi - -if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == 16'); then - format_error "FAILED: 16 PGS NOT UP" -fi +. `dirname $0`/run_3osds.sh try_change() { n=$1 s=$2 - $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":'$s',"pg_minsize":2,"pg_count":'$n',"failure_domain":"osd"}}' + $ETCDCTL put /vitastor/config/pools '{"1":{'$POOLCFG',"pg_size":'$s',"pg_minsize":'$PG_MINSIZE',"pg_count":'$n'}}' for i in {1..10}; do ($ETCDCTL get /vitastor/config/pgs --print-value-only |\ diff --git a/tests/test_heal.sh b/tests/test_heal.sh index fb3598da..c309f9cf 100755 --- a/tests/test_heal.sh +++ b/tests/test_heal.sh @@ -3,8 +3,10 @@ # Kill OSDs while writing PG_SIZE=3 - -. `dirname $0`/run_7osds.sh +OSD_COUNT=7 +PG_COUNT=32 +. `dirname $0`/run_3osds.sh +check_qemu IMG_SIZE=960 diff --git a/tests/test_interrupted_rebalance.sh b/tests/test_interrupted_rebalance.sh index e951d9e9..e0c06213 100755 --- a/tests/test_interrupted_rebalance.sh +++ b/tests/test_interrupted_rebalance.sh @@ -1,6 +1,8 @@ #!/bin/bash -ex -. `dirname $0`/run_7osds.sh +OSD_COUNT=7 +PG_COUNT=32 +. `dirname $0`/run_3osds.sh IMG_SIZE=960 @@ -38,7 +40,7 @@ fi # Check that no objects are lost ! nobj=`$ETCDCTL get --prefix '/vitastor/pg/stats' --print-value-only | jq -s '[ .[].object_count ] | reduce .[] as $num (0; .+$num)'` -if [ "$nobj" -ne $((IMG_SIZE*8)) ]; then +if [ "$nobj" -ne $((IMG_SIZE*8/PG_DATA_SIZE)) ]; then format_error "Data lost after multiple interrupted rebalancings" fi diff --git a/tests/test_rebalance_verify.sh b/tests/test_rebalance_verify.sh index 7f3d6bf2..23846f49 100755 --- a/tests/test_rebalance_verify.sh +++ b/tests/test_rebalance_verify.sh @@ -1,6 +1,8 @@ #!/bin/bash -ex -. `dirname $0`/run_7osds.sh +OSD_COUNT=7 +PG_COUNT=32 +. `dirname $0`/run_3osds.sh IMG_SIZE=256 diff --git a/tests/test_snapshot.sh b/tests/test_snapshot.sh index 88033735..661fe895 100755 --- a/tests/test_snapshot.sh +++ b/tests/test_snapshot.sh @@ -1,6 +1,7 @@ #!/bin/bash -ex . `dirname $0`/run_3osds.sh +check_qemu # Test basic write and snapshot diff --git a/tests/test_vm_start.sh b/tests/test_vm_start.sh index 5e3ec485..a5b0dec3 100755 --- a/tests/test_vm_start.sh +++ b/tests/test_vm_start.sh @@ -3,6 +3,7 @@ OSD_SIZE=2048 . `dirname $0`/run_3osds.sh +check_qemu $ETCDCTL put /vitastor/config/inode/1/1 '{"name":"debian9","size":'$((2048*1024*1024))'}' diff --git a/tests/test_write.sh b/tests/test_write.sh index 09fc6c68..c16b52e6 100755 --- a/tests/test_write.sh +++ b/tests/test_write.sh @@ -1,6 +1,7 @@ #!/bin/bash -ex . `dirname $0`/run_3osds.sh +check_qemu #LD_PRELOAD=libasan.so.5 \ # fio -thread -name=test -ioengine=build/src/libfio_vitastor_sec.so -bs=4k -fsync=128 `$ETCDCTL get /vitastor/osd/state/1 --print-value-only | jq -r '"-host="+.addresses[0]+" -port="+(.port|tostring)'` -rw=write -size=32M