khangvd
/
vitastor
forked from vitalif/vitastor
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Implement metadata dump tool

lrc-matrix
Vitaliy Filippov 2022-06-10 18:49:59 +03:00
parent 1eec4407ab
commit dac12d8a4c
11 changed files with 210 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
debian/vitastor-osd.install vendored
View File

@ -1,2 +1,3 @@
usr/bin/vitastor-osd
usr/bin/vitastor-dump-journal
usr/bin/vitastor-dump-meta

1
rpm/vitastor-el7.spec
View File

@ -113,6 +113,7 @@ cp -r mon %buildroot/usr/lib/vitastor
%files -n vitastor-osd
%_bindir/vitastor-osd
%_bindir/vitastor-dump-journal
%_bindir/vitastor-dump-meta
%files -n vitastor-mon

1
rpm/vitastor-el8.spec
View File

@ -110,6 +110,7 @@ cp -r mon %buildroot/usr/lib/vitastor
%files -n vitastor-osd
%_bindir/vitastor-osd
%_bindir/vitastor-dump-journal
%_bindir/vitastor-dump-meta
%files -n vitastor-mon

7
src/CMakeLists.txt
View File

@ -198,6 +198,11 @@ add_executable(vitastor-dump-journal
dump_journal.cpp crc32c.c
)
# vitastor-dump-meta
add_executable(vitastor-dump-meta
dump_meta.cpp
)
if (${WITH_QEMU})
# qemu_driver.so
add_library(qemu_vitastor SHARED
@ -275,7 +280,7 @@ target_include_directories(test_cluster_client PUBLIC ${CMAKE_SOURCE_DIR}/src/mo
### Install
install(TARGETS vitastor-osd vitastor-dump-journal vitastor-nbd vitastor-nfs vitastor-cli RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
install(TARGETS vitastor-osd vitastor-dump-journal vitastor-dump-meta vitastor-nbd vitastor-nfs vitastor-cli RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
install_symlink(vitastor-cli ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vitastor-rm)
install_symlink(vitastor-cli ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vita)
install(

6
src/blockstore.h
View File

@ -19,7 +19,11 @@
#include "timerfd_manager.h"
// Memory alignment for direct I/O (usually 512 bytes)
// All other alignments must be a multiple of this one
#ifndef DIRECT_IO_ALIGNMENT
#define DIRECT_IO_ALIGNMENT 512
#endif
// Memory allocation alignment (page size is usually optimal)
#ifndef MEM_ALIGNMENT
#define MEM_ALIGNMENT 4096
#endif

12
src/blockstore_open.cpp
View File

@ -109,25 +109,25 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config)
{
disk_alignment = 4096;
}
else if (disk_alignment % MEM_ALIGNMENT)
else if (disk_alignment % DIRECT_IO_ALIGNMENT)
{
throw std::runtime_error("disk_alignment must be a multiple of "+std::to_string(MEM_ALIGNMENT));
throw std::runtime_error("disk_alignment must be a multiple of "+std::to_string(DIRECT_IO_ALIGNMENT));
}
if (!journal_block_size)
{
journal_block_size = 4096;
}
else if (journal_block_size % MEM_ALIGNMENT)
else if (journal_block_size % DIRECT_IO_ALIGNMENT)
{
throw std::runtime_error("journal_block_size must be a multiple of "+std::to_string(MEM_ALIGNMENT));
throw std::runtime_error("journal_block_size must be a multiple of "+std::to_string(DIRECT_IO_ALIGNMENT));
}
if (!meta_block_size)
{
meta_block_size = 4096;
}
else if (meta_block_size % MEM_ALIGNMENT)
else if (meta_block_size % DIRECT_IO_ALIGNMENT)
{
throw std::runtime_error("meta_block_size must be a multiple of "+std::to_string(MEM_ALIGNMENT));
throw std::runtime_error("meta_block_size must be a multiple of "+std::to_string(DIRECT_IO_ALIGNMENT));
}
if (data_offset % disk_alignment)
{

5
src/cli_simple_offsets.cpp
View File

@ -14,6 +14,11 @@
std::function<bool(cli_result_t &)> cli_tool_t::simple_offsets(json11::Json cfg)
{
std::string device = cfg["device"].string_value();
if (device == "")
{
fprintf(stderr, "Device path is missing\n");
exit(1);
}
uint64_t object_size = parse_size(cfg["object_size"].string_value());
uint64_t bitmap_granularity = parse_size(cfg["bitmap_granularity"].string_value());
uint64_t journal_size = parse_size(cfg["journal_size"].string_value());

2
src/dump_journal.cpp
View File

@ -52,7 +52,7 @@ int main(int argc, char *argv[])
self.journal_block = strtoul(argv[b+1], NULL, 10);
self.journal_offset = strtoull(argv[b+2], NULL, 10);
self.journal_len = strtoull(argv[b+3], NULL, 10);
if (self.journal_block < MEM_ALIGNMENT || (self.journal_block % MEM_ALIGNMENT) ||
if (self.journal_block < DIRECT_IO_ALIGNMENT || (self.journal_block % DIRECT_IO_ALIGNMENT) ||
self.journal_block > 128*1024)
{
printf("Invalid journal block size\n");

173
src/dump_meta.cpp Normal file
View File

@ -0,0 +1,173 @@
// Copyright (c) Vitaliy Filippov, 2019+
// License: VNPL-1.1 (see README.md for details)
#define _LARGEFILE64_SOURCE
#include <sys/types.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/time.h>
#include <fcntl.h>
#include <unistd.h>
#include <stdint.h>
#include <malloc.h>
#include <linux/fs.h>
#include <string.h>
#include <errno.h>
#include <assert.h>
#include <stdio.h>
#include "object_id.h"
#include "osd_id.h"
// "VITAstor"
#define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
#define BLOCKSTORE_META_VERSION_V1 1
#define DIRECT_IO_ALIGNMENT 512
#define MEM_ALIGNMENT 4096
struct __attribute__((__packed__)) clean_disk_entry_v0_t
{
object_id oid;
uint64_t version;
uint8_t bitmap[];
};
struct __attribute__((__packed__)) blockstore_meta_header_v1_t
{
uint64_t zero;
uint64_t magic;
uint64_t version;
uint32_t meta_block_size;
uint32_t data_block_size;
uint32_t bitmap_granularity;
};
struct meta_dumper_t
{
char *meta_device;
uint32_t meta_block_size;
uint64_t meta_offset;
uint64_t meta_len;
uint64_t meta_pos;
int fd;
};
int main(int argc, char *argv[])
{
meta_dumper_t self = { 0 };
int b = 1;
if (argc < b+4)
{
printf("USAGE: %s <meta_file> <meta_block_size> <offset> <size>\n", argv[0]);
return 1;
}
self.meta_device = argv[b];
self.meta_block_size = strtoul(argv[b+1], NULL, 10);
self.meta_offset = strtoull(argv[b+2], NULL, 10);
self.meta_len = strtoull(argv[b+3], NULL, 10);
if (self.meta_block_size % DIRECT_IO_ALIGNMENT)
{
printf("Invalid metadata block size\n");
return 1;
}
self.fd = open(self.meta_device, O_DIRECT|O_RDONLY);
if (self.fd == -1)
{
printf("Failed to open metadata device\n");
return 1;
}
// Read all metadata into memory
void *data = memalign(MEM_ALIGNMENT, self.meta_len);
if (!data)
{
printf("Failed to allocate %lu MB of memory\n", self.meta_len/1024/1024);
close(self.fd);
return 1;
}
while (self.meta_pos < self.meta_len)
{
int r = pread(self.fd, data+self.meta_pos, self.meta_len-self.meta_pos, self.meta_offset+self.meta_pos);
assert(r > 0);
self.meta_pos += r;
}
close(self.fd);
// Check superblock
blockstore_meta_header_v1_t *hdr = (blockstore_meta_header_v1_t *)data;
if (hdr->zero == 0 &&
hdr->magic == BLOCKSTORE_META_MAGIC_V1 &&
hdr->version == BLOCKSTORE_META_VERSION_V1)
{
// Vitastor 0.6-0.7 - static array of clean_disk_entry_v0_t with bitmaps
if (hdr->meta_block_size != self.meta_block_size)
{
printf("Using block size %u bytes based on information from the superblock\n", hdr->meta_block_size);
self.meta_block_size = hdr->meta_block_size;
}
uint64_t clean_entry_bitmap_size = hdr->data_block_size / hdr->bitmap_granularity / 8;
uint64_t clean_entry_size = sizeof(clean_disk_entry_v0_t) + 2*clean_entry_bitmap_size;
uint64_t block_num = 0;
printf(
"{\"version\":\"0.6\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,\"entries\":[\n",
hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity
);
bool first = true;
for (uint64_t meta_pos = self.meta_block_size; meta_pos < self.meta_len; meta_pos += self.meta_block_size)
{
for (uint64_t ioff = 0; ioff < self.meta_block_size-clean_entry_size; ioff += clean_entry_size, block_num++)
{
clean_disk_entry_v0_t *entry = (clean_disk_entry_v0_t*)(data + meta_pos + ioff);
if (entry->oid.inode)
{
printf(
#define ENTRY_FMT "{\"block\":%lu,\"pool\":%u,\"inode\":%lu,\"stripe\":%lu,\"version\":%lu,\"bitmap\":\""
(first ? (",\n" ENTRY_FMT) : ENTRY_FMT),
#undef ENTRY_FMT
block_num, INODE_POOL(entry->oid.inode), INODE_NO_POOL(entry->oid.inode),
entry->oid.stripe, entry->version
);
first = false;
for (uint64_t i = 0; i < clean_entry_bitmap_size; i++)
{
printf("%02x", entry->bitmap[i]);
}
printf("\",\"ext_bitmap\":\"");
for (uint64_t i = 0; i < clean_entry_bitmap_size; i++)
{
printf("%02x", entry->bitmap[clean_entry_bitmap_size + i]);
}
printf("\"}");
}
}
}
printf("]}\n");
}
else
{
// Vitastor 0.4-0.5 - static array of clean_disk_entry_v0_t
uint64_t clean_entry_size = sizeof(clean_disk_entry_v0_t);
uint64_t block_num = 0;
printf("{\"version\":\"0.5\",\"meta_block_size\":%u,\"entries\":[\n", self.meta_block_size);
bool first = true;
for (uint64_t meta_pos = 0; meta_pos < self.meta_len; meta_pos += self.meta_block_size)
{
for (uint64_t ioff = 0; ioff < self.meta_block_size-clean_entry_size; ioff += clean_entry_size, block_num++)
{
clean_disk_entry_v0_t *entry = (clean_disk_entry_v0_t*)(data + meta_pos + ioff);
if (entry->oid.inode)
{
printf(
#define ENTRY_FMT "{\"block\":%lu,\"pool\":%u,\"inode\":%lu,\"stripe\":%lu,\"version\":%lu}"
(first ? (",\n" ENTRY_FMT) : ENTRY_FMT),
#undef ENTRY_FMT
block_num, INODE_POOL(entry->oid.inode), INODE_NO_POOL(entry->oid.inode),
entry->oid.stripe, entry->version
);
first = false;
}
}
}
printf("]}\n");
}
free(data);
}

14
src/osd_ops.h
View File

@ -30,13 +30,19 @@
#define OSD_OP_PING 15
#define OSD_OP_SEC_READ_BMP 16
#define OSD_OP_MAX 16
// Alignment & limit for read/write operations
#ifndef MEM_ALIGNMENT
#define MEM_ALIGNMENT 512
#endif
#define OSD_RW_MAX 64*1024*1024
#define OSD_PROTOCOL_VERSION 1
// Memory alignment for direct I/O (usually 512 bytes)
#ifndef DIRECT_IO_ALIGNMENT
#define DIRECT_IO_ALIGNMENT 512
#endif
// Memory allocation alignment (page size is usually optimal)
#ifndef MEM_ALIGNMENT
#define MEM_ALIGNMENT 4096
#endif
// common request and reply headers
struct __attribute__((__packed__)) osd_op_header_t
{

1
src/osd_rmw.h
View File

@ -7,6 +7,7 @@
#include "object_id.h"
#include "osd_id.h"
// Memory allocation alignment (page size is usually optimal)
#ifndef MEM_ALIGNMENT
#define MEM_ALIGNMENT 4096
#endif