parent
373f9d0387
commit
9b0db4f894
@ -0,0 +1,131 @@ |
||||
#!/bin/bash -ex |
||||
|
||||
# Run 3 OSDs |
||||
|
||||
. `dirname $0`/run_3osds.sh |
||||
|
||||
# Write inodes 1 and 2 |
||||
|
||||
LD_PRELOAD="build/src/libfio_vitastor.so" \ |
||||
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \ |
||||
-rw=write -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -runtime=10 |
||||
|
||||
LD_PRELOAD="build/src/libfio_vitastor.so" \ |
||||
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=1M -direct=1 -iodepth=4 \ |
||||
-rw=write -etcd=$ETCD_URL -pool=1 -inode=2 -size=128M -runtime=10 |
||||
|
||||
LD_PRELOAD="build/src/libfio_vitastor.so" \ |
||||
fio -thread -name=test -ioengine=build/src/libfio_vitastor.so -bs=4k -direct=1 -iodepth=16 \ |
||||
-rw=randwrite -etcd=$ETCD_URL -pool=1 -inode=1 -size=128M -runtime=10 &>/dev/null & |
||||
|
||||
sleep 5 |
||||
|
||||
# Stop OSD 1 |
||||
|
||||
kill -INT $OSD1_PID |
||||
sleep 2 |
||||
|
||||
# Remove inode 2 |
||||
|
||||
build/src/vitastor-cli rm-data --etcd_address $ETCD_URL --pool 1 --inode 2 |
||||
|
||||
# Run 3 more OSDs and move PG to 4,5,6 |
||||
|
||||
for i in $(seq 4 6); do |
||||
dd if=/dev/zero of=./testdata/test_osd$i.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1)) |
||||
build/src/vitastor-osd --osd_num $i --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd$i.bin 2>/dev/null) &>./testdata/osd$i.log & |
||||
eval OSD${i}_PID=$! |
||||
done |
||||
|
||||
$ETCDCTL put /vitastor/config/osd/1 '{"reweight":0}' |
||||
$ETCDCTL put /vitastor/config/osd/2 '{"reweight":0}' |
||||
$ETCDCTL put /vitastor/config/osd/3 '{"reweight":0}' |
||||
|
||||
# Wait for rebalance to finish |
||||
|
||||
wait_finish_rebalance() |
||||
{ |
||||
local sec=$1 |
||||
local st=$2 |
||||
local i=0 |
||||
while [[ $i -lt $sec ]]; do |
||||
if $ETCDCTL get --prefix /vitastor/pg/state/ --print-value-only | jq -s -e \ |
||||
'([ .[] | select(.state == ['$st'] and (.peers | contains([1]) | not) and (.peers | contains([2,3]) | not)) ] | length) == '$PG_COUNT; then |
||||
break |
||||
fi |
||||
sleep 1 |
||||
i=$((i+1)) |
||||
if [ $i -eq $sec ]; then |
||||
format_error "Rebalance couldn't finish in $sec seconds" |
||||
fi |
||||
done |
||||
} |
||||
|
||||
wait_finish_rebalance 60 '"active","left_on_dead"' |
||||
|
||||
# Stop OSD 2,3 |
||||
|
||||
kill -INT $OSD2_PID |
||||
kill -INT $OSD3_PID |
||||
sleep 2 |
||||
|
||||
# Verify that PGs are still active |
||||
|
||||
if ! ($ETCDCTL get /vitastor/pg/state/1/ --prefix --print-value-only | jq -s -e '[ .[] | select(.state == ["active","left_on_dead"]) ] | length == '$PG_COUNT); then |
||||
format_error "FAILED: $PG_COUNT PG(s) NOT UP" |
||||
fi |
||||
|
||||
# Start OSD 1 |
||||
|
||||
build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(build/src/vitastor-cli simple-offsets --format options ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log & |
||||
OSD1_PID=$! |
||||
|
||||
# Verify that inode 2 is removed and inode 1 is in place |
||||
|
||||
wait_repeer_1() |
||||
{ |
||||
local sec=$1 |
||||
local i=0 |
||||
while [[ $i -lt $sec ]]; do |
||||
if grep -q 'Repeer because of OSD 1' testdata/osd4.log testdata/osd5.log testdata/osd6.log; then |
||||
break |
||||
fi |
||||
sleep 1 |
||||
i=$((i+1)) |
||||
if [ $i -eq $sec ]; then |
||||
format_error "OSD 4/5/6 do not peer with older OSD 1" |
||||
fi |
||||
done |
||||
} |
||||
|
||||
wait_repeer_1 15 |
||||
|
||||
wait_finish_rebalance 15 '"active"' |
||||
|
||||
if [ "$SCHEME" = "replicated" ]; then |
||||
NOBJ=1024 |
||||
else |
||||
NOBJ=$((1024/(PG_SIZE-1))) |
||||
fi |
||||
|
||||
if ! ($ETCDCTL get /vitastor/pg/stats/1/1 --print-value-only | jq -s -e '.[0].object_count == '$NOBJ); then |
||||
format_error "FAILED: PG SHOULD CONTAIN EXACTLY 128 MB OF DATA, BUT IT DOESN'T" |
||||
fi |
||||
|
||||
qemu-img convert -S 4096 -p \ |
||||
-f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:pool=1:inode=1:size=4096" \ |
||||
-O raw ./testdata/inode1.bin |
||||
|
||||
qemu-img convert -S 4096 -p \ |
||||
-f raw "vitastor:etcd_host=127.0.0.1\:$ETCD_PORT/v3:pool=1:inode=2:size="$((128*1024*1024)) \ |
||||
-O raw ./testdata/inode2.bin |
||||
|
||||
if (dd if=/dev/zero bs=4096 count=1 | diff - ./testdata/inode1.bin); then |
||||
format_error "FAILED: INODE 1 SEEMS LOST" |
||||
fi |
||||
|
||||
if ! (dd if=/dev/zero bs=1M count=128 | diff - ./testdata/inode2.bin); then |
||||
format_error "FAILED: INODE 2 SEEMS RESTORED" |
||||
fi |
||||
|
||||
format_green OK |
||||
Loading…
Reference in new issue