From a420c77107041f9308ebb479dd6250fe1b41954f Mon Sep 17 00:00:00 2001
From: Vitaliy Filippov <vitalif@yourcmc.ru>
Date: Sun, 3 Apr 2022 01:35:29 +0300
Subject: [PATCH] Add rebalance-verify test

---
 tests/run_7osds.sh                  | 61 +++++++++++++++++++++++++++++
 tests/test_interrupted_rebalance.sh | 54 +------------------------
 tests/test_rebalance_verify.sh      | 42 ++++++++++++++++++++
 3 files changed, 105 insertions(+), 52 deletions(-)
 create mode 100644 tests/run_7osds.sh
 create mode 100755 tests/test_rebalance_verify.sh

diff --git a/tests/run_7osds.sh b/tests/run_7osds.sh
new file mode 100644
index 00000000..70bd7d05
--- /dev/null
+++ b/tests/run_7osds.sh
@@ -0,0 +1,61 @@
+#!/bin/bash
+
+. `dirname $0`/common.sh
+
+if [ "$IMMEDIATE_COMMIT" != "" ]; then
+    NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 1"
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5,"immediate_commit":"all"}'
+else
+    NO_SAME="--journal_sector_buffer_count 1024 --log_level 1"
+    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5}'
+fi
+
+OSD_SIZE=1024
+OSD_COUNT=7
+OSD_ARGS=
+for i in $(seq 1 $OSD_COUNT); do
+    dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
+    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $NO_SAME $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
+    eval OSD${i}_PID=$!
+done
+
+cd mon
+npm install
+cd ..
+node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log &
+MON_PID=$!
+
+$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":32,"failure_domain":"osd"}}'
+
+sleep 2
+
+if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == 32'); then
+    format_error "FAILED: 32 PGS NOT CONFIGURED"
+fi
+
+if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == 32'); then
+    format_error "FAILED: 32 PGS NOT UP"
+fi
+
+try_reweight()
+{
+    osd=$1
+    w=$2
+    $ETCDCTL put /vitastor/config/osd/$osd '{"reweight":'$w'}'
+    sleep 3
+}
+
+wait_finish_rebalance()
+{
+    sec=$1
+    i=0
+    while [[ $i -lt $sec ]]; do
+        ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == 32') && \
+            break
+        if [ $i -eq 60 ]; then
+            format_error "Rebalance couldn't finish in $sec seconds"
+        fi
+        sleep 1
+        i=$((i+1))
+    done
+}
diff --git a/tests/test_interrupted_rebalance.sh b/tests/test_interrupted_rebalance.sh
index 2b9ad010..cae69258 100755
--- a/tests/test_interrupted_rebalance.sh
+++ b/tests/test_interrupted_rebalance.sh
@@ -1,41 +1,6 @@
 #!/bin/bash -ex
 
-. `dirname $0`/common.sh
-
-if [ "$IMMEDIATE_COMMIT" != "" ]; then
-    NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all --log_level 1"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5,"immediate_commit":"all"}'
-else
-    NO_SAME="--journal_sector_buffer_count 1024 --log_level 1"
-    $ETCDCTL put /vitastor/config/global '{"recovery_queue_depth":1,"osd_out_time":5}'
-fi
-
-OSD_SIZE=1024
-OSD_COUNT=7
-OSD_ARGS=
-for i in $(seq 1 $OSD_COUNT); do
-    dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
-    build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
-    eval OSD${i}_PID=$!
-done
-
-cd mon
-npm install
-cd ..
-node mon/mon-main.js --etcd_url $ETCD_URL --etcd_prefix "/vitastor" --verbose 1 &>./testdata/mon.log &
-MON_PID=$!
-
-$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":32,"failure_domain":"osd"}}'
-
-sleep 2
-
-if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(.[0].items["1"] | map((.osd_set | select(. > 0)) | length == 2) | length) == 32'); then
-    format_error "FAILED: 32 PGS NOT CONFIGURED"
-fi
-
-if ! ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == 32'); then
-    format_error "FAILED: 32 PGS NOT UP"
-fi
+. `dirname $0`/run_7osds.sh
 
 IMG_SIZE=960
 
@@ -43,14 +8,6 @@ LD_PRELOAD=libasan.so.5 \
 fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4M -direct=1 -iodepth=16 -fsync=16 -rw=write \
     -etcd=$ETCD_URL -pool=1 -inode=2 -size=${IMG_SIZE}M -cluster_log_level=10
 
-try_reweight()
-{
-    osd=$1
-    w=$2
-    $ETCDCTL put /vitastor/config/osd/$osd '{"reweight":'$w'}'
-    sleep 3
-}
-
 try_reweight 1 0
 
 try_reweight 2 0
@@ -72,14 +29,7 @@ try_reweight 4 1
 try_reweight 5 1
 
 # Wait for the rebalance to finish
-for i in {1..60}; do
-    ($ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e '([ .[] | select(.state == ["active"]) ] | length) == 32') && \
-        break
-    if [ $i -eq 60 ]; then
-        format_error "Rebalance couldn't finish in 60 seconds"
-    fi
-    sleep 1
-done
+wait_finish_rebalance 60
 
 # Check that PGs never had degraded objects !
 if grep has_degraded ./testdata/mon.log; then
diff --git a/tests/test_rebalance_verify.sh b/tests/test_rebalance_verify.sh
new file mode 100755
index 00000000..3500ff6c
--- /dev/null
+++ b/tests/test_rebalance_verify.sh
@@ -0,0 +1,42 @@
+#!/bin/bash -ex
+
+. `dirname $0`/run_7osds.sh
+
+IMG_SIZE=256
+
+$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"testimg","size":'$((IMG_SIZE*1024*1024))'}'
+
+NBD_DEV=$(sudo build/src/vitastor-nbd map --etcd_address $ETCD_URL --image testimg --logfile ./testdata/nbd.log &)
+
+trap "sudo build/src/vitastor-nbd unmap $NBD_DEV" EXIT
+
+sudo chown $(id -u) $NBD_DEV
+
+dd if=/dev/urandom of=./testdata/img1.bin bs=1M count=$IMG_SIZE
+
+dd if=./testdata/img1.bin of=$NBD_DEV bs=1M count=$IMG_SIZE oflag=direct
+
+echo "Verifying before rebalance"
+dd if=$NBD_DEV of=./testdata/img2.bin bs=1M count=$IMG_SIZE iflag=direct
+diff ./testdata/img1.bin ./testdata/img2.bin
+
+try_reweight 1 0
+
+try_reweight 2 0
+
+for i in {1..10000}; do
+    O=$(((RANDOM*RANDOM) % (IMG_SIZE*128)))
+    dd if=$NBD_DEV of=./testdata/img2.bin bs=4k seek=$O skip=$O count=1 iflag=direct conv=notrunc
+done
+
+echo "Verifying during rebalance"
+diff ./testdata/img1.bin ./testdata/img2.bin
+
+# Wait for the rebalance to finish
+wait_finish_rebalance 60
+
+echo "Verifying after rebalance"
+dd if=$NBD_DEV of=./testdata/img2.bin bs=1M count=$IMG_SIZE iflag=direct
+diff ./testdata/img1.bin ./testdata/img2.bin
+
+format_green OK