132 lines
3.9 KiB
Bash
Executable File
132 lines
3.9 KiB
Bash
Executable File
#!/bin/bash -ex
|
|
|
|
# Run 3 OSDs
|
|
|
|
. `dirname $0`/run_3osds.sh
|
|
|
|
# Write inodes 1 and 2
|
|
|
|
LD_PRELOAD="build/src/libfio_vitastor.so" \
|
|
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \
|
|
-rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -runtime=10
|
|
|
|
LD_PRELOAD="build/src/libfio_vitastor.so" \
|
|
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \
|
|
-rw=write -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M -runtime=10
|
|
|
|
LD_PRELOAD="build/src/libfio_vitastor.so" \
|
|
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=16 \
|
|
-rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -runtime=10 &>/dev/null &
|
|
|
|
sleep 5
|
|
|
|
# Stop OSD 1
|
|
|
|
kill -INT $OSD1_PID
|
|
sleep 2
|
|
|
|
# Remove inode 2
|
|
|
|
build/src/vitastor-cli rm-data --etcd_address $ETCD_URL --pool 1 --inode 2
|
|
|
|
# Run 3 more OSDs and move PG to 4,5,6
|
|
|
|
for i in $(seq 4 6); do
|
|
dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
|
|
build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log &
|
|
eval OSD${i}_PID=$!
|
|
done
|
|
|
|
$ETCDCTL put /vitastor/config/osd/1 '{"reweight":0}'
|
|
$ETCDCTL put /vitastor/config/osd/2 '{"reweight":0}'
|
|
$ETCDCTL put /vitastor/config/osd/3 '{"reweight":0}'
|
|
|
|
# Wait for rebalance to finish
|
|
|
|
wait_finish_rebalance()
|
|
{
|
|
local sec=$1
|
|
local st=$2
|
|
local i=0
|
|
while [[ $i -lt $sec ]]; do
|
|
if $ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e \
|
|
'([ .[] | select(.state == ['$st'] and (.peers | contains([1]) | not) and (.peers | contains([2,3]) | not)) ] | length) == '$PG_COUNT; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
i=$((i+1))
|
|
if [ $i -eq $sec ]; then
|
|
format_error "Rebalance couldn't finish in $sec seconds"
|
|
fi
|
|
done
|
|
}
|
|
|
|
wait_finish_rebalance 60 '"active","left_on_dead"'
|
|
|
|
# Stop OSD 2,3
|
|
|
|
kill -INT $OSD2_PID
|
|
kill -INT $OSD3_PID
|
|
sleep 2
|
|
|
|
# Verify that PGs are still active
|
|
|
|
if ! ($ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active","left_on_dead"]) ] | length == '$PG_COUNT); then
|
|
format_error "FAILED: $PG_COUNT PG(s) NOT UP"
|
|
fi
|
|
|
|
# Start OSD 1
|
|
|
|
build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log &
|
|
OSD1_PID=$!
|
|
|
|
# Verify that inode 2 is removed and inode 1 is in place
|
|
|
|
wait_repeer_1()
|
|
{
|
|
local sec=$1
|
|
local i=0
|
|
while [[ $i -lt $sec ]]; do
|
|
if grep -q 'Repeer because of OSD 1' testdata/osd4.log testdata/osd5.log testdata/osd6.log; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
i=$((i+1))
|
|
if [ $i -eq $sec ]; then
|
|
format_error "OSD 4/5/6 do not peer with older OSD 1"
|
|
fi
|
|
done
|
|
}
|
|
|
|
wait_repeer_1 15
|
|
|
|
wait_finish_rebalance 15 '"active"'
|
|
|
|
if [ "$SCHEME" = "replicated" ]; then
|
|
NOBJ=1024
|
|
else
|
|
NOBJ=$((1024/(PG_SIZE-1)))
|
|
fi
|
|
|
|
if ! ($ETCDCTL get /vitastor/pg/stats/1/1 --print-value-only | jq -s -e '.[0].object_count == '$NOBJ); then
|
|
format_error "FAILED: PG SHOULD CONTAIN EXACTLY 128 MB OF DATA, BUT IT DOESN'T"
|
|
fi
|
|
|
|
qemu-img convert -S 4096 -p \
|
|
-f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:pool=1:inode=1:size=4096" \
|
|
-O raw ./testdata/inode1.bin
|
|
|
|
qemu-img convert -S 4096 -p \
|
|
-f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:pool=1:inode=2:size="$((128*1024*1024)) \
|
|
-O raw ./testdata/inode2.bin
|
|
|
|
if (dd if=/dev/zero bs=4096 count=1 | diff - ./testdata/inode1.bin); then
|
|
format_error "FAILED: INODE 1 SEEMS LOST"
|
|
fi
|
|
|
|
if ! (dd if=/dev/zero bs=1M count=128 | diff - ./testdata/inode2.bin); then
|
|
format_error "FAILED: INODE 2 SEEMS RESTORED"
|
|
fi
|
|
|
|
format_green OK
|