Implement metadata dump tool
parent
1eec4407ab
commit
dac12d8a4c
|
@ -1,2 +1,3 @@
|
||||||
usr/bin/vitastor-osd
|
usr/bin/vitastor-osd
|
||||||
usr/bin/vitastor-dump-journal
|
usr/bin/vitastor-dump-journal
|
||||||
|
usr/bin/vitastor-dump-meta
|
||||||
|
|
|
@ -113,6 +113,7 @@ cp -r mon %buildroot/usr/lib/vitastor
|
||||||
%files -n vitastor-osd
|
%files -n vitastor-osd
|
||||||
%_bindir/vitastor-osd
|
%_bindir/vitastor-osd
|
||||||
%_bindir/vitastor-dump-journal
|
%_bindir/vitastor-dump-journal
|
||||||
|
%_bindir/vitastor-dump-meta
|
||||||
|
|
||||||
|
|
||||||
%files -n vitastor-mon
|
%files -n vitastor-mon
|
||||||
|
|
|
@ -110,6 +110,7 @@ cp -r mon %buildroot/usr/lib/vitastor
|
||||||
%files -n vitastor-osd
|
%files -n vitastor-osd
|
||||||
%_bindir/vitastor-osd
|
%_bindir/vitastor-osd
|
||||||
%_bindir/vitastor-dump-journal
|
%_bindir/vitastor-dump-journal
|
||||||
|
%_bindir/vitastor-dump-meta
|
||||||
|
|
||||||
|
|
||||||
%files -n vitastor-mon
|
%files -n vitastor-mon
|
||||||
|
|
|
@ -198,6 +198,11 @@ add_executable(vitastor-dump-journal
|
||||||
dump_journal.cpp crc32c.c
|
dump_journal.cpp crc32c.c
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# vitastor-dump-meta
|
||||||
|
add_executable(vitastor-dump-meta
|
||||||
|
dump_meta.cpp
|
||||||
|
)
|
||||||
|
|
||||||
if (${WITH_QEMU})
|
if (${WITH_QEMU})
|
||||||
# qemu_driver.so
|
# qemu_driver.so
|
||||||
add_library(qemu_vitastor SHARED
|
add_library(qemu_vitastor SHARED
|
||||||
|
@ -275,7 +280,7 @@ target_include_directories(test_cluster_client PUBLIC ${CMAKE_SOURCE_DIR}/src/mo
|
||||||
|
|
||||||
### Install
|
### Install
|
||||||
|
|
||||||
install(TARGETS vitastor-osd vitastor-dump-journal vitastor-nbd vitastor-nfs vitastor-cli RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
|
install(TARGETS vitastor-osd vitastor-dump-journal vitastor-dump-meta vitastor-nbd vitastor-nfs vitastor-cli RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
|
||||||
install_symlink(vitastor-cli ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vitastor-rm)
|
install_symlink(vitastor-cli ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vitastor-rm)
|
||||||
install_symlink(vitastor-cli ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vita)
|
install_symlink(vitastor-cli ${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_BINDIR}/vita)
|
||||||
install(
|
install(
|
||||||
|
|
|
@ -19,7 +19,11 @@
|
||||||
#include "timerfd_manager.h"
|
#include "timerfd_manager.h"
|
||||||
|
|
||||||
// Memory alignment for direct I/O (usually 512 bytes)
|
// Memory alignment for direct I/O (usually 512 bytes)
|
||||||
// All other alignments must be a multiple of this one
|
#ifndef DIRECT_IO_ALIGNMENT
|
||||||
|
#define DIRECT_IO_ALIGNMENT 512
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// Memory allocation alignment (page size is usually optimal)
|
||||||
#ifndef MEM_ALIGNMENT
|
#ifndef MEM_ALIGNMENT
|
||||||
#define MEM_ALIGNMENT 4096
|
#define MEM_ALIGNMENT 4096
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -109,25 +109,25 @@ void blockstore_impl_t::parse_config(blockstore_config_t & config)
|
||||||
{
|
{
|
||||||
disk_alignment = 4096;
|
disk_alignment = 4096;
|
||||||
}
|
}
|
||||||
else if (disk_alignment % MEM_ALIGNMENT)
|
else if (disk_alignment % DIRECT_IO_ALIGNMENT)
|
||||||
{
|
{
|
||||||
throw std::runtime_error("disk_alignment must be a multiple of "+std::to_string(MEM_ALIGNMENT));
|
throw std::runtime_error("disk_alignment must be a multiple of "+std::to_string(DIRECT_IO_ALIGNMENT));
|
||||||
}
|
}
|
||||||
if (!journal_block_size)
|
if (!journal_block_size)
|
||||||
{
|
{
|
||||||
journal_block_size = 4096;
|
journal_block_size = 4096;
|
||||||
}
|
}
|
||||||
else if (journal_block_size % MEM_ALIGNMENT)
|
else if (journal_block_size % DIRECT_IO_ALIGNMENT)
|
||||||
{
|
{
|
||||||
throw std::runtime_error("journal_block_size must be a multiple of "+std::to_string(MEM_ALIGNMENT));
|
throw std::runtime_error("journal_block_size must be a multiple of "+std::to_string(DIRECT_IO_ALIGNMENT));
|
||||||
}
|
}
|
||||||
if (!meta_block_size)
|
if (!meta_block_size)
|
||||||
{
|
{
|
||||||
meta_block_size = 4096;
|
meta_block_size = 4096;
|
||||||
}
|
}
|
||||||
else if (meta_block_size % MEM_ALIGNMENT)
|
else if (meta_block_size % DIRECT_IO_ALIGNMENT)
|
||||||
{
|
{
|
||||||
throw std::runtime_error("meta_block_size must be a multiple of "+std::to_string(MEM_ALIGNMENT));
|
throw std::runtime_error("meta_block_size must be a multiple of "+std::to_string(DIRECT_IO_ALIGNMENT));
|
||||||
}
|
}
|
||||||
if (data_offset % disk_alignment)
|
if (data_offset % disk_alignment)
|
||||||
{
|
{
|
||||||
|
|
|
@ -14,6 +14,11 @@
|
||||||
std::function<bool(cli_result_t &)> cli_tool_t::simple_offsets(json11::Json cfg)
|
std::function<bool(cli_result_t &)> cli_tool_t::simple_offsets(json11::Json cfg)
|
||||||
{
|
{
|
||||||
std::string device = cfg["device"].string_value();
|
std::string device = cfg["device"].string_value();
|
||||||
|
if (device == "")
|
||||||
|
{
|
||||||
|
fprintf(stderr, "Device path is missing\n");
|
||||||
|
exit(1);
|
||||||
|
}
|
||||||
uint64_t object_size = parse_size(cfg["object_size"].string_value());
|
uint64_t object_size = parse_size(cfg["object_size"].string_value());
|
||||||
uint64_t bitmap_granularity = parse_size(cfg["bitmap_granularity"].string_value());
|
uint64_t bitmap_granularity = parse_size(cfg["bitmap_granularity"].string_value());
|
||||||
uint64_t journal_size = parse_size(cfg["journal_size"].string_value());
|
uint64_t journal_size = parse_size(cfg["journal_size"].string_value());
|
||||||
|
|
|
@ -52,7 +52,7 @@ int main(int argc, char *argv[])
|
||||||
self.journal_block = strtoul(argv[b+1], NULL, 10);
|
self.journal_block = strtoul(argv[b+1], NULL, 10);
|
||||||
self.journal_offset = strtoull(argv[b+2], NULL, 10);
|
self.journal_offset = strtoull(argv[b+2], NULL, 10);
|
||||||
self.journal_len = strtoull(argv[b+3], NULL, 10);
|
self.journal_len = strtoull(argv[b+3], NULL, 10);
|
||||||
if (self.journal_block < MEM_ALIGNMENT || (self.journal_block % MEM_ALIGNMENT) ||
|
if (self.journal_block < DIRECT_IO_ALIGNMENT || (self.journal_block % DIRECT_IO_ALIGNMENT) ||
|
||||||
self.journal_block > 128*1024)
|
self.journal_block > 128*1024)
|
||||||
{
|
{
|
||||||
printf("Invalid journal block size\n");
|
printf("Invalid journal block size\n");
|
||||||
|
|
|
@ -0,0 +1,173 @@
|
||||||
|
// Copyright (c) Vitaliy Filippov, 2019+
|
||||||
|
// License: VNPL-1.1 (see README.md for details)
|
||||||
|
|
||||||
|
#define _LARGEFILE64_SOURCE
|
||||||
|
#include <sys/types.h>
|
||||||
|
#include <sys/ioctl.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#include <sys/time.h>
|
||||||
|
#include <fcntl.h>
|
||||||
|
#include <unistd.h>
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <malloc.h>
|
||||||
|
#include <linux/fs.h>
|
||||||
|
#include <string.h>
|
||||||
|
#include <errno.h>
|
||||||
|
#include <assert.h>
|
||||||
|
#include <stdio.h>
|
||||||
|
|
||||||
|
#include "object_id.h"
|
||||||
|
#include "osd_id.h"
|
||||||
|
|
||||||
|
// "VITAstor"
|
||||||
|
#define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
|
||||||
|
#define BLOCKSTORE_META_VERSION_V1 1
|
||||||
|
|
||||||
|
#define DIRECT_IO_ALIGNMENT 512
|
||||||
|
#define MEM_ALIGNMENT 4096
|
||||||
|
|
||||||
|
struct __attribute__((__packed__)) clean_disk_entry_v0_t
|
||||||
|
{
|
||||||
|
object_id oid;
|
||||||
|
uint64_t version;
|
||||||
|
uint8_t bitmap[];
|
||||||
|
};
|
||||||
|
|
||||||
|
struct __attribute__((__packed__)) blockstore_meta_header_v1_t
|
||||||
|
{
|
||||||
|
uint64_t zero;
|
||||||
|
uint64_t magic;
|
||||||
|
uint64_t version;
|
||||||
|
uint32_t meta_block_size;
|
||||||
|
uint32_t data_block_size;
|
||||||
|
uint32_t bitmap_granularity;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct meta_dumper_t
|
||||||
|
{
|
||||||
|
char *meta_device;
|
||||||
|
uint32_t meta_block_size;
|
||||||
|
uint64_t meta_offset;
|
||||||
|
uint64_t meta_len;
|
||||||
|
uint64_t meta_pos;
|
||||||
|
int fd;
|
||||||
|
};
|
||||||
|
|
||||||
|
int main(int argc, char *argv[])
|
||||||
|
{
|
||||||
|
meta_dumper_t self = { 0 };
|
||||||
|
int b = 1;
|
||||||
|
if (argc < b+4)
|
||||||
|
{
|
||||||
|
printf("USAGE: %s <meta_file> <meta_block_size> <offset> <size>\n", argv[0]);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
self.meta_device = argv[b];
|
||||||
|
self.meta_block_size = strtoul(argv[b+1], NULL, 10);
|
||||||
|
self.meta_offset = strtoull(argv[b+2], NULL, 10);
|
||||||
|
self.meta_len = strtoull(argv[b+3], NULL, 10);
|
||||||
|
if (self.meta_block_size % DIRECT_IO_ALIGNMENT)
|
||||||
|
{
|
||||||
|
printf("Invalid metadata block size\n");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
self.fd = open(self.meta_device, O_DIRECT|O_RDONLY);
|
||||||
|
if (self.fd == -1)
|
||||||
|
{
|
||||||
|
printf("Failed to open metadata device\n");
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
// Read all metadata into memory
|
||||||
|
void *data = memalign(MEM_ALIGNMENT, self.meta_len);
|
||||||
|
if (!data)
|
||||||
|
{
|
||||||
|
printf("Failed to allocate %lu MB of memory\n", self.meta_len/1024/1024);
|
||||||
|
close(self.fd);
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
while (self.meta_pos < self.meta_len)
|
||||||
|
{
|
||||||
|
int r = pread(self.fd, data+self.meta_pos, self.meta_len-self.meta_pos, self.meta_offset+self.meta_pos);
|
||||||
|
assert(r > 0);
|
||||||
|
self.meta_pos += r;
|
||||||
|
}
|
||||||
|
close(self.fd);
|
||||||
|
// Check superblock
|
||||||
|
blockstore_meta_header_v1_t *hdr = (blockstore_meta_header_v1_t *)data;
|
||||||
|
if (hdr->zero == 0 &&
|
||||||
|
hdr->magic == BLOCKSTORE_META_MAGIC_V1 &&
|
||||||
|
hdr->version == BLOCKSTORE_META_VERSION_V1)
|
||||||
|
{
|
||||||
|
// Vitastor 0.6-0.7 - static array of clean_disk_entry_v0_t with bitmaps
|
||||||
|
if (hdr->meta_block_size != self.meta_block_size)
|
||||||
|
{
|
||||||
|
printf("Using block size %u bytes based on information from the superblock\n", hdr->meta_block_size);
|
||||||
|
self.meta_block_size = hdr->meta_block_size;
|
||||||
|
}
|
||||||
|
uint64_t clean_entry_bitmap_size = hdr->data_block_size / hdr->bitmap_granularity / 8;
|
||||||
|
uint64_t clean_entry_size = sizeof(clean_disk_entry_v0_t) + 2*clean_entry_bitmap_size;
|
||||||
|
uint64_t block_num = 0;
|
||||||
|
printf(
|
||||||
|
"{\"version\":\"0.6\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,\"entries\":[\n",
|
||||||
|
hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity
|
||||||
|
);
|
||||||
|
bool first = true;
|
||||||
|
for (uint64_t meta_pos = self.meta_block_size; meta_pos < self.meta_len; meta_pos += self.meta_block_size)
|
||||||
|
{
|
||||||
|
for (uint64_t ioff = 0; ioff < self.meta_block_size-clean_entry_size; ioff += clean_entry_size, block_num++)
|
||||||
|
{
|
||||||
|
clean_disk_entry_v0_t *entry = (clean_disk_entry_v0_t*)(data + meta_pos + ioff);
|
||||||
|
if (entry->oid.inode)
|
||||||
|
{
|
||||||
|
printf(
|
||||||
|
#define ENTRY_FMT "{\"block\":%lu,\"pool\":%u,\"inode\":%lu,\"stripe\":%lu,\"version\":%lu,\"bitmap\":\""
|
||||||
|
(first ? (",\n" ENTRY_FMT) : ENTRY_FMT),
|
||||||
|
#undef ENTRY_FMT
|
||||||
|
block_num, INODE_POOL(entry->oid.inode), INODE_NO_POOL(entry->oid.inode),
|
||||||
|
entry->oid.stripe, entry->version
|
||||||
|
);
|
||||||
|
first = false;
|
||||||
|
for (uint64_t i = 0; i < clean_entry_bitmap_size; i++)
|
||||||
|
{
|
||||||
|
printf("%02x", entry->bitmap[i]);
|
||||||
|
}
|
||||||
|
printf("\",\"ext_bitmap\":\"");
|
||||||
|
for (uint64_t i = 0; i < clean_entry_bitmap_size; i++)
|
||||||
|
{
|
||||||
|
printf("%02x", entry->bitmap[clean_entry_bitmap_size + i]);
|
||||||
|
}
|
||||||
|
printf("\"}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
printf("]}\n");
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
// Vitastor 0.4-0.5 - static array of clean_disk_entry_v0_t
|
||||||
|
uint64_t clean_entry_size = sizeof(clean_disk_entry_v0_t);
|
||||||
|
uint64_t block_num = 0;
|
||||||
|
printf("{\"version\":\"0.5\",\"meta_block_size\":%u,\"entries\":[\n", self.meta_block_size);
|
||||||
|
bool first = true;
|
||||||
|
for (uint64_t meta_pos = 0; meta_pos < self.meta_len; meta_pos += self.meta_block_size)
|
||||||
|
{
|
||||||
|
for (uint64_t ioff = 0; ioff < self.meta_block_size-clean_entry_size; ioff += clean_entry_size, block_num++)
|
||||||
|
{
|
||||||
|
clean_disk_entry_v0_t *entry = (clean_disk_entry_v0_t*)(data + meta_pos + ioff);
|
||||||
|
if (entry->oid.inode)
|
||||||
|
{
|
||||||
|
printf(
|
||||||
|
#define ENTRY_FMT "{\"block\":%lu,\"pool\":%u,\"inode\":%lu,\"stripe\":%lu,\"version\":%lu}"
|
||||||
|
(first ? (",\n" ENTRY_FMT) : ENTRY_FMT),
|
||||||
|
#undef ENTRY_FMT
|
||||||
|
block_num, INODE_POOL(entry->oid.inode), INODE_NO_POOL(entry->oid.inode),
|
||||||
|
entry->oid.stripe, entry->version
|
||||||
|
);
|
||||||
|
first = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
printf("]}\n");
|
||||||
|
}
|
||||||
|
free(data);
|
||||||
|
}
|
|
@ -30,13 +30,19 @@
|
||||||
#define OSD_OP_PING 15
|
#define OSD_OP_PING 15
|
||||||
#define OSD_OP_SEC_READ_BMP 16
|
#define OSD_OP_SEC_READ_BMP 16
|
||||||
#define OSD_OP_MAX 16
|
#define OSD_OP_MAX 16
|
||||||
// Alignment & limit for read/write operations
|
|
||||||
#ifndef MEM_ALIGNMENT
|
|
||||||
#define MEM_ALIGNMENT 512
|
|
||||||
#endif
|
|
||||||
#define OSD_RW_MAX 64*1024*1024
|
#define OSD_RW_MAX 64*1024*1024
|
||||||
#define OSD_PROTOCOL_VERSION 1
|
#define OSD_PROTOCOL_VERSION 1
|
||||||
|
|
||||||
|
// Memory alignment for direct I/O (usually 512 bytes)
|
||||||
|
#ifndef DIRECT_IO_ALIGNMENT
|
||||||
|
#define DIRECT_IO_ALIGNMENT 512
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// Memory allocation alignment (page size is usually optimal)
|
||||||
|
#ifndef MEM_ALIGNMENT
|
||||||
|
#define MEM_ALIGNMENT 4096
|
||||||
|
#endif
|
||||||
|
|
||||||
// common request and reply headers
|
// common request and reply headers
|
||||||
struct __attribute__((__packed__)) osd_op_header_t
|
struct __attribute__((__packed__)) osd_op_header_t
|
||||||
{
|
{
|
||||||
|
|
|
@ -7,6 +7,7 @@
|
||||||
#include "object_id.h"
|
#include "object_id.h"
|
||||||
#include "osd_id.h"
|
#include "osd_id.h"
|
||||||
|
|
||||||
|
// Memory allocation alignment (page size is usually optimal)
|
||||||
#ifndef MEM_ALIGNMENT
|
#ifndef MEM_ALIGNMENT
|
||||||
#define MEM_ALIGNMENT 4096
|
#define MEM_ALIGNMENT 4096
|
||||||
#endif
|
#endif
|
||||||
|
|
Loading…
Reference in New Issue