Compare commits
3 Commits
f9fe72d40a
...
3de553ecd7
Author | SHA1 | Date |
---|---|---|
Vitaliy Filippov | 3de553ecd7 | |
Vitaliy Filippov | 9c45d43e74 | |
Vitaliy Filippov | 891250d355 |
|
@ -200,6 +200,14 @@ target_link_libraries(osd_peering_pg_test tcmalloc_minimal)
|
||||||
# test_allocator
|
# test_allocator
|
||||||
add_executable(test_allocator test_allocator.cpp allocator.cpp)
|
add_executable(test_allocator test_allocator.cpp allocator.cpp)
|
||||||
|
|
||||||
|
# test_cas
|
||||||
|
add_executable(test_cas
|
||||||
|
test_cas.cpp
|
||||||
|
)
|
||||||
|
target_link_libraries(test_cas
|
||||||
|
vitastor_client
|
||||||
|
)
|
||||||
|
|
||||||
# test_cluster_client
|
# test_cluster_client
|
||||||
add_executable(test_cluster_client
|
add_executable(test_cluster_client
|
||||||
test_cluster_client.cpp
|
test_cluster_client.cpp
|
||||||
|
|
|
@ -633,6 +633,13 @@ resume_1:
|
||||||
// Slice the operation into parts
|
// Slice the operation into parts
|
||||||
slice_rw(op);
|
slice_rw(op);
|
||||||
op->needs_reslice = false;
|
op->needs_reslice = false;
|
||||||
|
if (op->opcode == OSD_OP_WRITE && op->version && op->parts.size() > 1)
|
||||||
|
{
|
||||||
|
// Atomic writes to multiple stripes are unsupported
|
||||||
|
op->retval = -EINVAL;
|
||||||
|
erase_op(op);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
resume_2:
|
resume_2:
|
||||||
// Send unsent parts, if they're not subject to change
|
// Send unsent parts, if they're not subject to change
|
||||||
op->state = 3;
|
op->state = 3;
|
||||||
|
@ -922,6 +929,7 @@ bool cluster_client_t::try_send(cluster_op_t *op, int i)
|
||||||
.offset = part->offset,
|
.offset = part->offset,
|
||||||
.len = part->len,
|
.len = part->len,
|
||||||
.meta_revision = meta_rev,
|
.meta_revision = meta_rev,
|
||||||
|
.version = op->opcode == OSD_OP_WRITE ? op->version : 0,
|
||||||
} },
|
} },
|
||||||
.bitmap = op->opcode == OSD_OP_WRITE ? NULL : op->part_bitmaps + pg_bitmap_size*i,
|
.bitmap = op->opcode == OSD_OP_WRITE ? NULL : op->part_bitmaps + pg_bitmap_size*i,
|
||||||
.bitmap_len = (unsigned)(op->opcode == OSD_OP_WRITE ? 0 : pg_bitmap_size),
|
.bitmap_len = (unsigned)(op->opcode == OSD_OP_WRITE ? 0 : pg_bitmap_size),
|
||||||
|
@ -1072,10 +1080,6 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
|
||||||
if (part->op.reply.hdr.retval != expected)
|
if (part->op.reply.hdr.retval != expected)
|
||||||
{
|
{
|
||||||
// Operation failed, retry
|
// Operation failed, retry
|
||||||
fprintf(
|
|
||||||
stderr, "%s operation failed on OSD %lu: retval=%ld (expected %d), dropping connection\n",
|
|
||||||
osd_op_names[part->op.req.hdr.opcode], part->osd_num, part->op.reply.hdr.retval, expected
|
|
||||||
);
|
|
||||||
if (part->op.reply.hdr.retval == -EPIPE)
|
if (part->op.reply.hdr.retval == -EPIPE)
|
||||||
{
|
{
|
||||||
// Mark op->up_wait = true before stopping the client
|
// Mark op->up_wait = true before stopping the client
|
||||||
|
@ -1094,7 +1098,14 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
|
||||||
// Don't overwrite other errors with -EPIPE
|
// Don't overwrite other errors with -EPIPE
|
||||||
op->retval = part->op.reply.hdr.retval;
|
op->retval = part->op.reply.hdr.retval;
|
||||||
}
|
}
|
||||||
msgr.stop_client(part->op.peer_fd);
|
if (op->retval != -EINTR && op->retval != -EIO)
|
||||||
|
{
|
||||||
|
fprintf(
|
||||||
|
stderr, "%s operation failed on OSD %lu: retval=%ld (expected %d), dropping connection\n",
|
||||||
|
osd_op_names[part->op.req.hdr.opcode], part->osd_num, part->op.reply.hdr.retval, expected
|
||||||
|
);
|
||||||
|
msgr.stop_client(part->op.peer_fd);
|
||||||
|
}
|
||||||
part->flags |= PART_ERROR;
|
part->flags |= PART_ERROR;
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
@ -1106,6 +1117,7 @@ void cluster_client_t::handle_op_part(cluster_op_part_t *part)
|
||||||
if (op->opcode == OSD_OP_READ)
|
if (op->opcode == OSD_OP_READ)
|
||||||
{
|
{
|
||||||
copy_part_bitmap(op, part);
|
copy_part_bitmap(op, part);
|
||||||
|
op->version = op->parts.size() == 1 ? part->op.reply.rw.version : 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (op->inflight_count == 0)
|
if (op->inflight_count == 0)
|
||||||
|
|
|
@ -31,6 +31,9 @@ struct cluster_op_t
|
||||||
uint64_t inode;
|
uint64_t inode;
|
||||||
uint64_t offset;
|
uint64_t offset;
|
||||||
uint64_t len;
|
uint64_t len;
|
||||||
|
// for reads and writes within a single object (stripe),
|
||||||
|
// reads can return current version and writes can use "CAS" semantics
|
||||||
|
uint64_t version = 0;
|
||||||
int retval;
|
int retval;
|
||||||
osd_op_buf_list_t iov;
|
osd_op_buf_list_t iov;
|
||||||
std::function<void(cluster_op_t*)> callback;
|
std::function<void(cluster_op_t*)> callback;
|
||||||
|
|
|
@ -261,7 +261,7 @@ void osd_messenger_t::try_connect_peer_addr(osd_num_t peer_osd, const char *peer
|
||||||
{
|
{
|
||||||
osd_num_t peer_osd = clients.at(peer_fd)->osd_num;
|
osd_num_t peer_osd = clients.at(peer_fd)->osd_num;
|
||||||
stop_client(peer_fd, true);
|
stop_client(peer_fd, true);
|
||||||
on_connect_peer(peer_osd, -EIO);
|
on_connect_peer(peer_osd, -EPIPE);
|
||||||
return;
|
return;
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
|
@ -191,6 +191,9 @@ struct __attribute__((__packed__)) osd_op_rw_t
|
||||||
uint32_t flags;
|
uint32_t flags;
|
||||||
// inode metadata revision
|
// inode metadata revision
|
||||||
uint64_t meta_revision;
|
uint64_t meta_revision;
|
||||||
|
// object version for atomic "CAS" (compare-and-set) writes
|
||||||
|
// writes and deletes fail with -EINTR if object version differs from (version-1)
|
||||||
|
uint64_t version;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct __attribute__((__packed__)) osd_reply_rw_t
|
struct __attribute__((__packed__)) osd_reply_rw_t
|
||||||
|
@ -199,6 +202,8 @@ struct __attribute__((__packed__)) osd_reply_rw_t
|
||||||
// for reads: bitmap length
|
// for reads: bitmap length
|
||||||
uint32_t bitmap_len;
|
uint32_t bitmap_len;
|
||||||
uint32_t pad0;
|
uint32_t pad0;
|
||||||
|
// for reads: object version
|
||||||
|
uint64_t version;
|
||||||
};
|
};
|
||||||
|
|
||||||
// sync to the primary OSD
|
// sync to the primary OSD
|
||||||
|
|
|
@ -222,6 +222,7 @@ resume_2:
|
||||||
finish_op(cur_op, op_data->epipe > 0 ? -EPIPE : -EIO);
|
finish_op(cur_op, op_data->epipe > 0 ? -EPIPE : -EIO);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
cur_op->reply.rw.version = op_data->fact_ver;
|
||||||
cur_op->reply.rw.bitmap_len = op_data->pg_data_size * clean_entry_bitmap_size;
|
cur_op->reply.rw.bitmap_len = op_data->pg_data_size * clean_entry_bitmap_size;
|
||||||
if (op_data->degraded)
|
if (op_data->degraded)
|
||||||
{
|
{
|
||||||
|
@ -343,6 +344,12 @@ resume_3:
|
||||||
pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->epipe > 0 ? -EPIPE : -EIO);
|
pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->epipe > 0 ? -EPIPE : -EIO);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
// Check CAS version
|
||||||
|
if (cur_op->req.rw.version && op_data->fact_ver != (cur_op->req.rw.version-1))
|
||||||
|
{
|
||||||
|
cur_op->reply.hdr.retval = -EINTR;
|
||||||
|
goto continue_others;
|
||||||
|
}
|
||||||
// Save version override for parallel reads
|
// Save version override for parallel reads
|
||||||
pg.ver_override[op_data->oid] = op_data->fact_ver;
|
pg.ver_override[op_data->oid] = op_data->fact_ver;
|
||||||
// Submit deletes
|
// Submit deletes
|
||||||
|
@ -370,6 +377,8 @@ resume_5:
|
||||||
free_object_state(pg, &op_data->object_state);
|
free_object_state(pg, &op_data->object_state);
|
||||||
}
|
}
|
||||||
pg.total_count--;
|
pg.total_count--;
|
||||||
|
cur_op->reply.hdr.retval = 0;
|
||||||
|
continue_others:
|
||||||
osd_op_t *next_op = NULL;
|
osd_op_t *next_op = NULL;
|
||||||
auto next_it = pg.write_queue.find(op_data->oid);
|
auto next_it = pg.write_queue.find(op_data->oid);
|
||||||
if (next_it != pg.write_queue.end() && next_it->second == cur_op)
|
if (next_it != pg.write_queue.end() && next_it->second == cur_op)
|
||||||
|
@ -378,7 +387,7 @@ resume_5:
|
||||||
if (next_it != pg.write_queue.end() && next_it->first == op_data->oid)
|
if (next_it != pg.write_queue.end() && next_it->first == op_data->oid)
|
||||||
next_op = next_it->second;
|
next_op = next_it->second;
|
||||||
}
|
}
|
||||||
finish_op(cur_op, cur_op->req.rw.len);
|
finish_op(cur_op, cur_op->reply.hdr.retval);
|
||||||
if (next_op)
|
if (next_op)
|
||||||
{
|
{
|
||||||
// Continue next write to the same object
|
// Continue next write to the same object
|
||||||
|
|
|
@ -65,7 +65,10 @@ int osd_t::read_bitmaps(osd_op_t *cur_op, pg_t & pg, int base_state)
|
||||||
auto vo_it = pg.ver_override.find(cur_oid);
|
auto vo_it = pg.ver_override.find(cur_oid);
|
||||||
auto read_version = (vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX);
|
auto read_version = (vo_it != pg.ver_override.end() ? vo_it->second : UINT64_MAX);
|
||||||
// Read bitmap synchronously from the local database
|
// Read bitmap synchronously from the local database
|
||||||
bs->read_bitmap(cur_oid, read_version, op_data->snapshot_bitmaps + chain_num*clean_entry_bitmap_size, NULL);
|
bs->read_bitmap(
|
||||||
|
cur_oid, read_version, op_data->snapshot_bitmaps + chain_num*clean_entry_bitmap_size,
|
||||||
|
!chain_num ? &cur_op->reply.rw.version : NULL
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
@ -228,7 +231,10 @@ int osd_t::submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg)
|
||||||
// Read bitmap synchronously from the local database
|
// Read bitmap synchronously from the local database
|
||||||
for (int j = prev; j <= i; j++)
|
for (int j = prev; j <= i; j++)
|
||||||
{
|
{
|
||||||
bs->read_bitmap((*bitmap_requests)[j].oid, (*bitmap_requests)[j].version, (*bitmap_requests)[j].bmp_buf, NULL);
|
bs->read_bitmap(
|
||||||
|
(*bitmap_requests)[j].oid, (*bitmap_requests)[j].version, (*bitmap_requests)[j].bmp_buf,
|
||||||
|
(*bitmap_requests)[j].oid.inode == cur_op->req.rw.inode ? &cur_op->reply.rw.version : NULL
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
|
@ -264,6 +270,10 @@ int osd_t::submit_bitmap_subops(osd_op_t *cur_op, pg_t & pg)
|
||||||
for (int j = prev; j <= i; j++)
|
for (int j = prev; j <= i; j++)
|
||||||
{
|
{
|
||||||
memcpy((*bitmap_requests)[j].bmp_buf, cur_buf, clean_entry_bitmap_size);
|
memcpy((*bitmap_requests)[j].bmp_buf, cur_buf, clean_entry_bitmap_size);
|
||||||
|
if ((*bitmap_requests)[j].oid.inode == cur_op->req.rw.inode)
|
||||||
|
{
|
||||||
|
memcpy(&cur_op->reply.rw.version, cur_buf-8, 8);
|
||||||
|
}
|
||||||
cur_buf += 8 + clean_entry_bitmap_size;
|
cur_buf += 8 + clean_entry_bitmap_size;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -96,6 +96,12 @@ resume_3:
|
||||||
pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->epipe > 0 ? -EPIPE : -EIO);
|
pg_cancel_write_queue(pg, cur_op, op_data->oid, op_data->epipe > 0 ? -EPIPE : -EIO);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
// Check CAS version
|
||||||
|
if (cur_op->req.rw.version && op_data->fact_ver != (cur_op->req.rw.version-1))
|
||||||
|
{
|
||||||
|
cur_op->reply.hdr.retval = -EINTR;
|
||||||
|
goto continue_others;
|
||||||
|
}
|
||||||
if (op_data->scheme == POOL_SCHEME_REPLICATED)
|
if (op_data->scheme == POOL_SCHEME_REPLICATED)
|
||||||
{
|
{
|
||||||
// Set bitmap bits
|
// Set bitmap bits
|
||||||
|
@ -265,7 +271,7 @@ continue_others:
|
||||||
next_op = next_it->second;
|
next_op = next_it->second;
|
||||||
}
|
}
|
||||||
// finish_op would invalidate next_it if it cleared pg.write_queue, but it doesn't do that :)
|
// finish_op would invalidate next_it if it cleared pg.write_queue, but it doesn't do that :)
|
||||||
finish_op(cur_op, cur_op->req.rw.len);
|
finish_op(cur_op, cur_op->reply.hdr.retval);
|
||||||
if (next_op)
|
if (next_op)
|
||||||
{
|
{
|
||||||
// Continue next write to the same object
|
// Continue next write to the same object
|
||||||
|
|
|
@ -0,0 +1,135 @@
|
||||||
|
// Copyright (c) Vitaliy Filippov, 2019+
|
||||||
|
// License: VNPL-1.1 (see README.md for details)
|
||||||
|
|
||||||
|
#include <stdio.h>
|
||||||
|
#include <stdlib.h>
|
||||||
|
|
||||||
|
#include "epoll_manager.h"
|
||||||
|
#include "cluster_client.h"
|
||||||
|
|
||||||
|
void send_read(cluster_client_t *cli, uint64_t inode, std::function<void(int, uint64_t)> cb)
|
||||||
|
{
|
||||||
|
cluster_op_t *op = new cluster_op_t();
|
||||||
|
op->opcode = OSD_OP_READ;
|
||||||
|
op->inode = inode;
|
||||||
|
op->offset = 0;
|
||||||
|
op->len = 4096;
|
||||||
|
op->iov.push_back(malloc_or_die(op->len), op->len);
|
||||||
|
op->callback = [cb](cluster_op_t *op)
|
||||||
|
{
|
||||||
|
uint64_t version = op->version;
|
||||||
|
int retval = op->retval;
|
||||||
|
if (retval == op->len)
|
||||||
|
retval = 0;
|
||||||
|
free(op->iov.buf[0].iov_base);
|
||||||
|
delete op;
|
||||||
|
if (cb != NULL)
|
||||||
|
cb(retval, version);
|
||||||
|
};
|
||||||
|
cli->execute(op);
|
||||||
|
}
|
||||||
|
|
||||||
|
void send_write(cluster_client_t *cli, uint64_t inode, int byte, uint64_t version, std::function<void(int)> cb)
|
||||||
|
{
|
||||||
|
cluster_op_t *op = new cluster_op_t();
|
||||||
|
op->opcode = OSD_OP_WRITE;
|
||||||
|
op->inode = inode;
|
||||||
|
op->offset = 0;
|
||||||
|
op->len = 4096;
|
||||||
|
op->version = version;
|
||||||
|
op->iov.push_back(malloc_or_die(op->len), op->len);
|
||||||
|
memset(op->iov.buf[0].iov_base, byte, op->len);
|
||||||
|
op->callback = [cb](cluster_op_t *op)
|
||||||
|
{
|
||||||
|
int retval = op->retval;
|
||||||
|
if (retval == op->len)
|
||||||
|
retval = 0;
|
||||||
|
free(op->iov.buf[0].iov_base);
|
||||||
|
delete op;
|
||||||
|
if (cb != NULL)
|
||||||
|
cb(retval);
|
||||||
|
};
|
||||||
|
cli->execute(op);
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(int narg, char *args[])
|
||||||
|
{
|
||||||
|
json11::Json::object cfgo;
|
||||||
|
for (int i = 1; i < narg; i++)
|
||||||
|
{
|
||||||
|
if (args[i][0] == '-' && args[i][1] == '-')
|
||||||
|
{
|
||||||
|
const char *opt = args[i]+2;
|
||||||
|
cfgo[opt] = i == narg-1 ? "1" : args[++i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
json11::Json cfg(cfgo);
|
||||||
|
uint64_t inode = (cfg["pool_id"].uint64_value() << (64-POOL_ID_BITS))
|
||||||
|
| cfg["inode_id"].uint64_value();
|
||||||
|
uint64_t base_ver = 0;
|
||||||
|
// Create client
|
||||||
|
auto ringloop = new ring_loop_t(512);
|
||||||
|
auto epmgr = new epoll_manager_t(ringloop);
|
||||||
|
auto cli = new cluster_client_t(ringloop, epmgr->tfd, cfg);
|
||||||
|
cli->on_ready([&]()
|
||||||
|
{
|
||||||
|
send_read(cli, inode, [&](int r, uint64_t v)
|
||||||
|
{
|
||||||
|
if (r < 0)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Initial read operation failed\n");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
base_ver = v;
|
||||||
|
// CAS v=1 = compare with zero, non-existing object
|
||||||
|
send_write(cli, inode, 0x01, base_ver+1, [&](int r)
|
||||||
|
{
|
||||||
|
if (r < 0)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "CAS for non-existing object failed\n");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
// Check that read returns the new version
|
||||||
|
send_read(cli, inode, [&](int r, uint64_t v)
|
||||||
|
{
|
||||||
|
if (r < 0)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Read operation failed after write\n");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
if (v != base_ver+1)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Read operation failed to return the new version number\n");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
// CAS v=2 = compare with v=1, existing object
|
||||||
|
send_write(cli, inode, 0x02, base_ver+2, [&](int r)
|
||||||
|
{
|
||||||
|
if (r < 0)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "CAS for existing object failed\n");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
// CAS v=2 again = compare with v=1, but version is 2. Must fail with -EINTR
|
||||||
|
send_write(cli, inode, 0x03, base_ver+2, [&](int r)
|
||||||
|
{
|
||||||
|
if (r != -EINTR)
|
||||||
|
{
|
||||||
|
fprintf(stderr, "CAS conflict detection failed\n");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
|
printf("Basic CAS test succeeded\n");
|
||||||
|
exit(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
|
while (1)
|
||||||
|
{
|
||||||
|
ringloop->loop();
|
||||||
|
ringloop->wait();
|
||||||
|
}
|
||||||
|
return 0;
|
||||||
|
}
|
|
@ -0,0 +1,43 @@
|
||||||
|
#!/bin/bash -ex
|
||||||
|
|
||||||
|
. `dirname $0`/common.sh
|
||||||
|
|
||||||
|
OSD_SIZE=${OSD_SIZE:-1024}
|
||||||
|
|
||||||
|
dd if=/dev/zero of=./testdata/test_osd1.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
|
||||||
|
dd if=/dev/zero of=./testdata/test_osd2.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
|
||||||
|
dd if=/dev/zero of=./testdata/test_osd3.bin bs=1024 count=1 seek=$((OSD_SIZE*1024-1))
|
||||||
|
|
||||||
|
build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log &
|
||||||
|
OSD1_PID=$!
|
||||||
|
build/src/vitastor-osd --osd_num 2 --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd2.bin 2>/dev/null) &>./testdata/osd2.log &
|
||||||
|
OSD2_PID=$!
|
||||||
|
build/src/vitastor-osd --osd_num 3 --bind_address 127.0.0.1 $OSD_ARGS --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd3.bin 2>/dev/null) &>./testdata/osd3.log &
|
||||||
|
OSD3_PID=$!
|
||||||
|
|
||||||
|
cd mon
|
||||||
|
npm install
|
||||||
|
cd ..
|
||||||
|
node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
|
||||||
|
MON_PID=$!
|
||||||
|
|
||||||
|
if [ -n "$GLOBAL_CONF" ]; then
|
||||||
|
$ETCDCTL put /vitastor/config/global "$GLOBAL_CONF"
|
||||||
|
fi
|
||||||
|
|
||||||
|
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1,"pg_count":1,"failure_domain":"osd"}}'
|
||||||
|
|
||||||
|
sleep 2
|
||||||
|
|
||||||
|
if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and (.[0].items["1"]["1"].osd_set | sort) == ["1","2","3"]'); then
|
||||||
|
format_error "FAILED: 1 PG NOT CONFIGURED"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then
|
||||||
|
format_error "FAILED: 1 PG NOT UP"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so; then
|
||||||
|
sudo rm -f /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
|
||||||
|
sudo ln -s "$(realpath .)/build/src/block-vitastor.so" /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
|
||||||
|
fi
|
|
@ -0,0 +1,7 @@
|
||||||
|
#!/bin/bash -ex
|
||||||
|
|
||||||
|
. `dirname $0`/run_3osds.sh
|
||||||
|
|
||||||
|
build/src/test_cas --pool_id 1 --inode_id 1 --etcd_address $ETCD_URL
|
||||||
|
|
||||||
|
format_green OK
|
|
@ -1,40 +1,6 @@
|
||||||
#!/bin/bash -ex
|
#!/bin/bash -ex
|
||||||
|
|
||||||
. `dirname $0`/common.sh
|
. `dirname $0`/run_3osds.sh
|
||||||
|
|
||||||
dd if=/dev/zero of=./testdata/test_osd1.bin bs=1024 count=1 seek=$((1024*1024-1))
|
|
||||||
dd if=/dev/zero of=./testdata/test_osd2.bin bs=1024 count=1 seek=$((1024*1024-1))
|
|
||||||
dd if=/dev/zero of=./testdata/test_osd3.bin bs=1024 count=1 seek=$((1024*1024-1))
|
|
||||||
|
|
||||||
build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log &
|
|
||||||
OSD1_PID=$!
|
|
||||||
build/src/vitastor-osd --osd_num 2 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd2.bin 2>/dev/null) &>./testdata/osd2.log &
|
|
||||||
OSD2_PID=$!
|
|
||||||
build/src/vitastor-osd --osd_num 3 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd3.bin 2>/dev/null) &>./testdata/osd3.log &
|
|
||||||
OSD3_PID=$!
|
|
||||||
|
|
||||||
cd mon
|
|
||||||
npm install
|
|
||||||
cd ..
|
|
||||||
node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
|
|
||||||
MON_PID=$!
|
|
||||||
|
|
||||||
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1,"pg_count":1,"failure_domain":"osd"}}'
|
|
||||||
|
|
||||||
sleep 2
|
|
||||||
|
|
||||||
if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and (.[0].items["1"]["1"].osd_set | sort) == ["1","2","3"]'); then
|
|
||||||
format_error "FAILED: 1 PG NOT CONFIGURED"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then
|
|
||||||
format_error "FAILED: 1 PG NOT UP"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so; then
|
|
||||||
sudo rm -f /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
|
|
||||||
sudo ln -s "$(realpath .)/build/src/block-vitastor.so" /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Test basic write and snapshot
|
# Test basic write and snapshot
|
||||||
|
|
||||||
|
|
|
@ -1,40 +1,8 @@
|
||||||
#!/bin/bash -ex
|
#!/bin/bash -ex
|
||||||
|
|
||||||
. `dirname $0`/common.sh
|
OSD_SIZE=2048
|
||||||
|
|
||||||
dd if=/dev/zero of=./testdata/test_osd1.bin bs=2048 count=1 seek=$((1024*1024-1))
|
. `dirname $0`/run_3osds.sh
|
||||||
dd if=/dev/zero of=./testdata/test_osd2.bin bs=2048 count=1 seek=$((1024*1024-1))
|
|
||||||
dd if=/dev/zero of=./testdata/test_osd3.bin bs=2048 count=1 seek=$((1024*1024-1))
|
|
||||||
|
|
||||||
build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log &
|
|
||||||
OSD1_PID=$!
|
|
||||||
build/src/vitastor-osd --osd_num 2 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd2.bin 2>/dev/null) &>./testdata/osd2.log &
|
|
||||||
OSD2_PID=$!
|
|
||||||
build/src/vitastor-osd --osd_num 3 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd3.bin 2>/dev/null) &>./testdata/osd3.log &
|
|
||||||
OSD3_PID=$!
|
|
||||||
|
|
||||||
cd mon
|
|
||||||
npm install
|
|
||||||
cd ..
|
|
||||||
node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
|
|
||||||
MON_PID=$!
|
|
||||||
|
|
||||||
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1,"pg_count":1,"failure_domain":"osd"}}'
|
|
||||||
|
|
||||||
sleep 2
|
|
||||||
|
|
||||||
if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and (.[0].items["1"]["1"].osd_set | sort) == ["1","2","3"]'); then
|
|
||||||
format_error "FAILED: 1 PG NOT CONFIGURED"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then
|
|
||||||
format_error "FAILED: 1 PG NOT UP"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so; then
|
|
||||||
sudo rm -f /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
|
|
||||||
sudo ln -s "$(realpath .)/build/src/block-vitastor.so" /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
|
|
||||||
fi
|
|
||||||
|
|
||||||
$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"debian9","size":'$((2048*1024*1024))'}'
|
$ETCDCTL put /vitastor/config/inode/1/1 '{"name":"debian9","size":'$((2048*1024*1024))'}'
|
||||||
|
|
||||||
|
|
|
@ -1,44 +1,10 @@
|
||||||
#!/bin/bash -ex
|
#!/bin/bash -ex
|
||||||
|
|
||||||
. `dirname $0`/common.sh
|
. `dirname $0`/run_3osds.sh
|
||||||
|
|
||||||
dd if=/dev/zero of=./testdata/test_osd1.bin bs=1024 count=1 seek=$((1024*1024-1))
|
|
||||||
dd if=/dev/zero of=./testdata/test_osd2.bin bs=1024 count=1 seek=$((1024*1024-1))
|
|
||||||
dd if=/dev/zero of=./testdata/test_osd3.bin bs=1024 count=1 seek=$((1024*1024-1))
|
|
||||||
|
|
||||||
build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log &
|
|
||||||
OSD1_PID=$!
|
|
||||||
build/src/vitastor-osd --osd_num 2 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd2.bin 2>/dev/null) &>./testdata/osd2.log &
|
|
||||||
OSD2_PID=$!
|
|
||||||
build/src/vitastor-osd --osd_num 3 --bind_address 127.0.0.1 --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd3.bin 2>/dev/null) &>./testdata/osd3.log &
|
|
||||||
OSD3_PID=$!
|
|
||||||
|
|
||||||
cd mon
|
|
||||||
npm install
|
|
||||||
cd ..
|
|
||||||
node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
|
|
||||||
MON_PID=$!
|
|
||||||
|
|
||||||
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1,"pg_count":1,"failure_domain":"osd"}}'
|
|
||||||
|
|
||||||
sleep 2
|
|
||||||
|
|
||||||
if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and (.[0].items["1"]["1"].osd_set | sort) == ["1","2","3"]'); then
|
|
||||||
format_error "FAILED: 1 PG NOT CONFIGURED"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then
|
|
||||||
format_error "FAILED: 1 PG NOT UP"
|
|
||||||
fi
|
|
||||||
|
|
||||||
#LD_PRELOAD=libasan.so.5 \
|
#LD_PRELOAD=libasan.so.5 \
|
||||||
# fio -thread -name=test -ioengine=build/src/libfio_vitastor_sec.so -bs=4k -fsync=128 `$ETCDCTL get /vitastor/osd/state/1 --print-value-only | jq -r '"-host="+.addresses[0]+" -port="+(.port|tostring)'` -rw=write -size=32M
|
# fio -thread -name=test -ioengine=build/src/libfio_vitastor_sec.so -bs=4k -fsync=128 `$ETCDCTL get /vitastor/osd/state/1 --print-value-only | jq -r '"-host="+.addresses[0]+" -port="+(.port|tostring)'` -rw=write -size=32M
|
||||||
|
|
||||||
if ! cmp build/src/block-vitastor.so /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so; then
|
|
||||||
sudo rm -f /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
|
|
||||||
sudo ln -s "$(realpath .)/build/src/block-vitastor.so" /usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so
|
|
||||||
fi
|
|
||||||
|
|
||||||
# A lot of parallel syncs was crashing the primary OSD at some point
|
# A lot of parallel syncs was crashing the primary OSD at some point
|
||||||
|
|
||||||
LD_PRELOAD=libasan.so.5 \
|
LD_PRELOAD=libasan.so.5 \
|
||||||
|
|
|
@ -1,40 +1,10 @@
|
||||||
#!/bin/bash -ex
|
#!/bin/bash -ex
|
||||||
# Test the `no_same_sector_overwrites` mode
|
# Test the `no_same_sector_overwrites` mode
|
||||||
|
|
||||||
. `dirname $0`/common.sh
|
OSD_ARGS="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all"
|
||||||
|
GLOBAL_CONF='{"immediate_commit":"all"}'
|
||||||
|
|
||||||
dd if=/dev/zero of=./testdata/test_osd1.bin bs=1024 count=1 seek=$((1024*1024-1))
|
. `dirname $0`/run_3osds.sh
|
||||||
dd if=/dev/zero of=./testdata/test_osd2.bin bs=1024 count=1 seek=$((1024*1024-1))
|
|
||||||
dd if=/dev/zero of=./testdata/test_osd3.bin bs=1024 count=1 seek=$((1024*1024-1))
|
|
||||||
|
|
||||||
NO_SAME="--journal_no_same_sector_overwrites true --journal_sector_buffer_count 1024 --disable_data_fsync 1 --immediate_commit all"
|
|
||||||
|
|
||||||
build/src/vitastor-osd --osd_num 1 --bind_address 127.0.0.1 $NO_SAME --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd1.bin 2>/dev/null) &>./testdata/osd1.log &
|
|
||||||
OSD1_PID=$!
|
|
||||||
build/src/vitastor-osd --osd_num 2 --bind_address 127.0.0.1 $NO_SAME --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd2.bin 2>/dev/null) &>./testdata/osd2.log &
|
|
||||||
OSD2_PID=$!
|
|
||||||
build/src/vitastor-osd --osd_num 3 --bind_address 127.0.0.1 $NO_SAME --etcd_address $ETCD_URL $(node mon/simple-offsets.js --format options --device ./testdata/test_osd3.bin 2>/dev/null) &>./testdata/osd3.log &
|
|
||||||
OSD3_PID=$!
|
|
||||||
|
|
||||||
cd mon
|
|
||||||
npm install
|
|
||||||
cd ..
|
|
||||||
node mon/mon-main.js --etcd_url http://$ETCD_URL --etcd_prefix "/vitastor" &>./testdata/mon.log &
|
|
||||||
MON_PID=$!
|
|
||||||
|
|
||||||
$ETCDCTL put /vitastor/config/global '{"immediate_commit":"all"}'
|
|
||||||
|
|
||||||
$ETCDCTL put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"xor","pg_size":3,"pg_minsize":2,"parity_chunks":1,"pg_count":1,"failure_domain":"osd"}}'
|
|
||||||
|
|
||||||
sleep 2
|
|
||||||
|
|
||||||
if ! ($ETCDCTL get /vitastor/config/pgs --print-value-only | jq -s -e '(. | length) != 0 and (.[0].items["1"]["1"].osd_set | sort) == ["1","2","3"]'); then
|
|
||||||
format_error "FAILED: 1 PG NOT CONFIGURED"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if ! ($ETCDCTL get /vitastor/pg/state/1/1 --print-value-only | jq -s -e '(. | length) != 0 and .[0].state == ["active"]'); then
|
|
||||||
format_error "FAILED: 1 PG NOT UP"
|
|
||||||
fi
|
|
||||||
|
|
||||||
#LSAN_OPTIONS=report_objects=true:suppressions=`pwd`/testdata/lsan-suppress.txt LD_PRELOAD=libasan.so.5 \
|
#LSAN_OPTIONS=report_objects=true:suppressions=`pwd`/testdata/lsan-suppress.txt LD_PRELOAD=libasan.so.5 \
|
||||||
# fio -thread -name=test -ioengine=build/src/libfio_vitastor_sec.so -bs=4k -fsync=128 `$ETCDCTL get /vitastor/osd/state/1 --print-value-only | jq -r '"-host="+.addresses[0]+" -port="+(.port|tostring)'` -rw=write -size=32M
|
# fio -thread -name=test -ioengine=build/src/libfio_vitastor_sec.so -bs=4k -fsync=128 `$ETCDCTL get /vitastor/osd/state/1 --print-value-only | jq -r '"-host="+.addresses[0]+" -port="+(.port|tostring)'` -rw=write -size=32M
|
||||||
|
|
Loading…
Reference in New Issue