diff --git a/tests/common.sh b/tests/common.sh index d54af3ba..7879beab 100644 --- a/tests/common.sh +++ b/tests/common.sh @@ -23,20 +23,40 @@ trap 'kill -9 $(jobs -p)' EXIT ETCD=${ETCD:-etcd} ETCD_IP=${ETCD_IP:-127.0.0.1} ETCD_PORT=${ETCD_PORT:-12379} +ETCD_COUNT=${ETCD_COUNT:-1} if [ "$KEEP_DATA" = "" ]; then rm -rf ./testdata mkdir -p ./testdata fi -$ETCD -name etcd_test --data-dir ./testdata/etcd \ - --advertise-client-urls http://$ETCD_IP:$ETCD_PORT --listen-client-urls http://$ETCD_IP:$ETCD_PORT \ - --initial-advertise-peer-urls http://$ETCD_IP:$((ETCD_PORT+1)) --listen-peer-urls http://$ETCD_IP:$((ETCD_PORT+1)) \ - --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision &>./testdata/etcd.log & -ETCD_PID=$! -ETCD_URL=$ETCD_IP:$ETCD_PORT/v3 +ETCD_URL="http://$ETCD_IP:$ETCD_PORT" +ETCD_CLUSTER="etcd1=http://$ETCD_IP:$((ETCD_PORT+1))" +for i in $(seq 2 $ETCD_COUNT); do + ETCD_URL="$ETCD_URL,http://$ETCD_IP:$((ETCD_PORT+2*i-2))" + ETCD_CLUSTER="$ETCD_CLUSTER,etcd$i=http://$ETCD_IP:$((ETCD_PORT+2*i-1))" +done ETCDCTL="${ETCD}ctl --endpoints=http://$ETCD_URL" +start_etcd() +{ + local i=$1 + $ETCD -name etcd$i --data-dir ./testdata/etcd$i \ + --advertise-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) --listen-client-urls http://$ETCD_IP:$((ETCD_PORT+2*i-2)) \ + --initial-advertise-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) --listen-peer-urls http://$ETCD_IP:$((ETCD_PORT+2*i-1)) \ + --initial-cluster-token vitastor-tests-etcd --initial-cluster-state new \ + --initial-cluster "$ETCD_CLUSTER" \ + --max-txn-ops=100000 --auto-compaction-retention=10 --auto-compaction-mode=revision &>./testdata/etcd$i.log & + eval ETCD${i}_PID=$! +} + +for i in $(seq 1 $ETCD_COUNT); do + start_etcd $i +done +if [ $ETCD_COUNT -gt 1 ]; then + sleep 1 +fi + echo leak:fio >> testdata/lsan-suppress.txt echo leak:tcmalloc >> testdata/lsan-suppress.txt echo leak:ceph >> testdata/lsan-suppress.txt diff --git a/tests/run_3osds.sh b/tests/run_3osds.sh index 9c99db21..4766a9e9 100644 --- a/tests/run_3osds.sh +++ b/tests/run_3osds.sh @@ -18,7 +18,7 @@ done cd mon npm install cd .. -node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & +node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & MON_PID=$! if [ -n "$GLOBAL_CONF" ]; then diff --git a/tests/test_change_pg_count.sh b/tests/test_change_pg_count.sh index 2a8cbac8..008f0c4f 100755 --- a/tests/test_change_pg_count.sh +++ b/tests/test_change_pg_count.sh @@ -22,7 +22,7 @@ done cd mon npm install cd .. -node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log & +node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log & MON_PID=$! $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool",'$POOLCFG',"pg_count":16,"failure_domain":"osd"}}' diff --git a/tests/test_change_pg_size.sh b/tests/test_change_pg_size.sh index bca5f340..a4f10dce 100755 --- a/tests/test_change_pg_size.sh +++ b/tests/test_change_pg_size.sh @@ -14,7 +14,7 @@ done cd mon npm install cd .. -node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & +node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & MON_PID=$! $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":3,"pg_minsize":2,"pg_count":16,"failure_domain":"osd"}}' diff --git a/tests/test_etcd_fail.sh b/tests/test_etcd_fail.sh new file mode 100755 index 00000000..a18162d9 --- /dev/null +++ b/tests/test_etcd_fail.sh @@ -0,0 +1,33 @@ +#!/bin/bash -ex + +# Run 5 etcds and kill them while operating +ETCD_COUNT=5 + +. `dirname $0`/run_3osds.sh + +LD_PRELOAD=libasan.so.5 \ + fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \ + -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10 + +kill_etcds() +{ + sleep 5 + kill $ETCD1_PID $ETCD2_PID + sleep 5 + kill $ETCD3_PID + start_etcd 1 + sleep 5 + kill $ETCD4_PID + start_etcd 2 + sleep 5 + kill $ETCD5_PID + start_etcd 3 +} + +kill_etcds & + +LD_PRELOAD=libasan.so.5 \ + fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=1 -fsync=1 -rw=randwrite \ + -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -cluster_log_level=10 -runtime=30 + +format_green OK diff --git a/tests/test_failure_domain.sh b/tests/test_failure_domain.sh index bc245d79..8432b176 100755 --- a/tests/test_failure_domain.sh +++ b/tests/test_failure_domain.sh @@ -18,7 +18,7 @@ $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicate cd mon npm install cd .. -node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & +node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & MON_PID=$! sleep 2 diff --git a/tests/test_interrupted_rebalance.sh b/tests/test_interrupted_rebalance.sh index c8defccf..2b9ad010 100755 --- a/tests/test_interrupted_rebalance.sh +++ b/tests/test_interrupted_rebalance.sh @@ -22,7 +22,7 @@ done cd mon npm install cd .. -node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log & +node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log & MON_PID=$! $ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":32,"failure_domain":"osd"}}' diff --git a/tests/test_vm_cont.sh b/tests/test_vm_cont.sh index dfb6b850..2f554156 100755 --- a/tests/test_vm_cont.sh +++ b/tests/test_vm_cont.sh @@ -14,7 +14,7 @@ for i in $(seq 1 $OSD_COUNT); do eval OSD${i}_PID=$! done -node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & +node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log & MON_PID=$! sleep 3