forked from vitalif/vitastor
Do not read all metadata into memory when dumping
parent
0876ca09cd
commit
bce357e2a5
|
@ -200,7 +200,7 @@ add_executable(vitastor-dump-journal
|
||||||
|
|
||||||
# vitastor-dump-meta
|
# vitastor-dump-meta
|
||||||
add_executable(vitastor-dump-meta
|
add_executable(vitastor-dump-meta
|
||||||
dump_meta.cpp
|
dump_meta.cpp rw_blocking.cpp
|
||||||
)
|
)
|
||||||
|
|
||||||
if (${WITH_QEMU})
|
if (${WITH_QEMU})
|
||||||
|
|
|
@ -90,13 +90,13 @@
|
||||||
#include "blockstore_journal.h"
|
#include "blockstore_journal.h"
|
||||||
|
|
||||||
// "VITAstor"
|
// "VITAstor"
|
||||||
#define BLOCKSTORE_META_MAGIC 0x726F747341544956l
|
#define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
|
||||||
#define BLOCKSTORE_META_VERSION 1
|
#define BLOCKSTORE_META_VERSION_V1 1
|
||||||
|
|
||||||
// metadata header (superblock)
|
// metadata header (superblock)
|
||||||
// FIXME: After adding the OSD superblock, add a key to metadata
|
// FIXME: After adding the OSD superblock, add a key to metadata
|
||||||
// and journal headers to check if they belong to the same OSD
|
// and journal headers to check if they belong to the same OSD
|
||||||
struct __attribute__((__packed__)) blockstore_meta_header_t
|
struct __attribute__((__packed__)) blockstore_meta_header_v1_t
|
||||||
{
|
{
|
||||||
uint64_t zero;
|
uint64_t zero;
|
||||||
uint64_t magic;
|
uint64_t magic;
|
||||||
|
|
|
@ -71,10 +71,10 @@ resume_1:
|
||||||
if (iszero((uint64_t*)metadata_buffer, bs->meta_block_size / sizeof(uint64_t)))
|
if (iszero((uint64_t*)metadata_buffer, bs->meta_block_size / sizeof(uint64_t)))
|
||||||
{
|
{
|
||||||
{
|
{
|
||||||
blockstore_meta_header_t *hdr = (blockstore_meta_header_t *)metadata_buffer;
|
blockstore_meta_header_v1_t *hdr = (blockstore_meta_header_v1_t *)metadata_buffer;
|
||||||
hdr->zero = 0;
|
hdr->zero = 0;
|
||||||
hdr->magic = BLOCKSTORE_META_MAGIC;
|
hdr->magic = BLOCKSTORE_META_MAGIC_V1;
|
||||||
hdr->version = BLOCKSTORE_META_VERSION;
|
hdr->version = BLOCKSTORE_META_VERSION_V1;
|
||||||
hdr->meta_block_size = bs->meta_block_size;
|
hdr->meta_block_size = bs->meta_block_size;
|
||||||
hdr->data_block_size = bs->block_size;
|
hdr->data_block_size = bs->block_size;
|
||||||
hdr->bitmap_granularity = bs->bitmap_granularity;
|
hdr->bitmap_granularity = bs->bitmap_granularity;
|
||||||
|
@ -103,10 +103,10 @@ resume_1:
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
blockstore_meta_header_t *hdr = (blockstore_meta_header_t *)metadata_buffer;
|
blockstore_meta_header_v1_t *hdr = (blockstore_meta_header_v1_t *)metadata_buffer;
|
||||||
if (hdr->zero != 0 ||
|
if (hdr->zero != 0 ||
|
||||||
hdr->magic != BLOCKSTORE_META_MAGIC ||
|
hdr->magic != BLOCKSTORE_META_MAGIC_V1 ||
|
||||||
hdr->version != BLOCKSTORE_META_VERSION)
|
hdr->version != BLOCKSTORE_META_VERSION_V1)
|
||||||
{
|
{
|
||||||
printf(
|
printf(
|
||||||
"Metadata is corrupt or old version.\n"
|
"Metadata is corrupt or old version.\n"
|
||||||
|
|
|
@ -12,32 +12,9 @@
|
||||||
#include <assert.h>
|
#include <assert.h>
|
||||||
#include <stdio.h>
|
#include <stdio.h>
|
||||||
|
|
||||||
#include "object_id.h"
|
#include "blockstore_impl.h"
|
||||||
#include "osd_id.h"
|
#include "osd_id.h"
|
||||||
|
#include "rw_blocking.h"
|
||||||
// "VITAstor"
|
|
||||||
#define BLOCKSTORE_META_MAGIC_V1 0x726F747341544956l
|
|
||||||
#define BLOCKSTORE_META_VERSION_V1 1
|
|
||||||
|
|
||||||
#define DIRECT_IO_ALIGNMENT 512
|
|
||||||
#define MEM_ALIGNMENT 4096
|
|
||||||
|
|
||||||
struct __attribute__((__packed__)) clean_disk_entry_v0_t
|
|
||||||
{
|
|
||||||
object_id oid;
|
|
||||||
uint64_t version;
|
|
||||||
uint8_t bitmap[];
|
|
||||||
};
|
|
||||||
|
|
||||||
struct __attribute__((__packed__)) blockstore_meta_header_v1_t
|
|
||||||
{
|
|
||||||
uint64_t zero;
|
|
||||||
uint64_t magic;
|
|
||||||
uint64_t version;
|
|
||||||
uint32_t meta_block_size;
|
|
||||||
uint32_t data_block_size;
|
|
||||||
uint32_t bitmap_granularity;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct meta_dumper_t
|
struct meta_dumper_t
|
||||||
{
|
{
|
||||||
|
@ -47,6 +24,8 @@ struct meta_dumper_t
|
||||||
uint64_t meta_len;
|
uint64_t meta_len;
|
||||||
uint64_t meta_pos;
|
uint64_t meta_pos;
|
||||||
int fd;
|
int fd;
|
||||||
|
|
||||||
|
int dump();
|
||||||
};
|
};
|
||||||
|
|
||||||
int main(int argc, char *argv[])
|
int main(int argc, char *argv[])
|
||||||
|
@ -62,108 +41,129 @@ int main(int argc, char *argv[])
|
||||||
self.meta_block_size = strtoul(argv[b+1], NULL, 10);
|
self.meta_block_size = strtoul(argv[b+1], NULL, 10);
|
||||||
self.meta_offset = strtoull(argv[b+2], NULL, 10);
|
self.meta_offset = strtoull(argv[b+2], NULL, 10);
|
||||||
self.meta_len = strtoull(argv[b+3], NULL, 10);
|
self.meta_len = strtoull(argv[b+3], NULL, 10);
|
||||||
if (self.meta_block_size % DIRECT_IO_ALIGNMENT)
|
return self.dump();
|
||||||
|
}
|
||||||
|
|
||||||
|
int meta_dumper_t::dump()
|
||||||
|
{
|
||||||
|
if (this->meta_block_size % DIRECT_IO_ALIGNMENT)
|
||||||
{
|
{
|
||||||
printf("Invalid metadata block size\n");
|
printf("Invalid metadata block size\n");
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
self.fd = open(self.meta_device, O_DIRECT|O_RDONLY);
|
this->fd = open(this->meta_device, O_DIRECT|O_RDONLY);
|
||||||
if (self.fd == -1)
|
if (this->fd == -1)
|
||||||
{
|
{
|
||||||
printf("Failed to open metadata device\n");
|
printf("Failed to open metadata device\n");
|
||||||
return 1;
|
return 1;
|
||||||
}
|
}
|
||||||
// Read all metadata into memory
|
int buf_size = 1024*1024;
|
||||||
void *data = memalign(MEM_ALIGNMENT, self.meta_len);
|
if (buf_size % this->meta_block_size)
|
||||||
if (!data)
|
buf_size = 8*this->meta_block_size;
|
||||||
{
|
if (buf_size > this->meta_len)
|
||||||
printf("Failed to allocate %lu MB of memory\n", self.meta_len/1024/1024);
|
buf_size = this->meta_len;
|
||||||
close(self.fd);
|
void *data = memalign_or_die(MEM_ALIGNMENT, buf_size);
|
||||||
return 1;
|
lseek64(this->fd, this->meta_offset, 0);
|
||||||
}
|
read_blocking(this->fd, data, buf_size);
|
||||||
while (self.meta_pos < self.meta_len)
|
|
||||||
{
|
|
||||||
int r = pread(self.fd, data+self.meta_pos, self.meta_len-self.meta_pos, self.meta_offset+self.meta_pos);
|
|
||||||
assert(r > 0);
|
|
||||||
self.meta_pos += r;
|
|
||||||
}
|
|
||||||
close(self.fd);
|
|
||||||
// Check superblock
|
// Check superblock
|
||||||
blockstore_meta_header_v1_t *hdr = (blockstore_meta_header_v1_t *)data;
|
blockstore_meta_header_v1_t *hdr = (blockstore_meta_header_v1_t *)data;
|
||||||
if (hdr->zero == 0 &&
|
if (hdr->zero == 0 &&
|
||||||
hdr->magic == BLOCKSTORE_META_MAGIC_V1 &&
|
hdr->magic == BLOCKSTORE_META_MAGIC_V1 &&
|
||||||
hdr->version == BLOCKSTORE_META_VERSION_V1)
|
hdr->version == BLOCKSTORE_META_VERSION_V1)
|
||||||
{
|
{
|
||||||
// Vitastor 0.6-0.7 - static array of clean_disk_entry_v0_t with bitmaps
|
// Vitastor 0.6-0.7 - static array of clean_disk_entry with bitmaps
|
||||||
if (hdr->meta_block_size != self.meta_block_size)
|
if (hdr->meta_block_size != this->meta_block_size)
|
||||||
{
|
{
|
||||||
printf("Using block size %u bytes based on information from the superblock\n", hdr->meta_block_size);
|
printf("Using block size of %u bytes based on information from the superblock\n", hdr->meta_block_size);
|
||||||
self.meta_block_size = hdr->meta_block_size;
|
this->meta_block_size = hdr->meta_block_size;
|
||||||
|
if (buf_size % this->meta_block_size)
|
||||||
|
{
|
||||||
|
buf_size = 8*this->meta_block_size;
|
||||||
|
free(data);
|
||||||
|
data = memalign_or_die(MEM_ALIGNMENT, buf_size);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
this->meta_offset += this->meta_block_size;
|
||||||
|
this->meta_len -= this->meta_block_size;
|
||||||
uint64_t clean_entry_bitmap_size = hdr->data_block_size / hdr->bitmap_granularity / 8;
|
uint64_t clean_entry_bitmap_size = hdr->data_block_size / hdr->bitmap_granularity / 8;
|
||||||
uint64_t clean_entry_size = sizeof(clean_disk_entry_v0_t) + 2*clean_entry_bitmap_size;
|
uint64_t clean_entry_size = sizeof(clean_disk_entry) + 2*clean_entry_bitmap_size;
|
||||||
uint64_t block_num = 0;
|
uint64_t block_num = 0;
|
||||||
printf(
|
printf(
|
||||||
"{\"version\":\"0.6\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,\"entries\":[\n",
|
"{\"version\":\"0.6\",\"meta_block_size\":%u,\"data_block_size\":%u,\"bitmap_granularity\":%u,\"entries\":[\n",
|
||||||
hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity
|
hdr->meta_block_size, hdr->data_block_size, hdr->bitmap_granularity
|
||||||
);
|
);
|
||||||
bool first = true;
|
bool first = true;
|
||||||
for (uint64_t meta_pos = self.meta_block_size; meta_pos < self.meta_len; meta_pos += self.meta_block_size)
|
lseek64(this->fd, this->meta_offset, 0);
|
||||||
|
while (this->meta_pos < this->meta_len)
|
||||||
{
|
{
|
||||||
for (uint64_t ioff = 0; ioff < self.meta_block_size-clean_entry_size; ioff += clean_entry_size, block_num++)
|
uint64_t read_len = buf_size < this->meta_len-this->meta_pos ? buf_size : this->meta_len-this->meta_pos;
|
||||||
|
read_blocking(this->fd, data, read_len);
|
||||||
|
this->meta_pos += read_len;
|
||||||
|
for (uint64_t blk = 0; blk < read_len; blk += this->meta_block_size)
|
||||||
{
|
{
|
||||||
clean_disk_entry_v0_t *entry = (clean_disk_entry_v0_t*)(data + meta_pos + ioff);
|
for (uint64_t ioff = 0; ioff < this->meta_block_size-clean_entry_size; ioff += clean_entry_size, block_num++)
|
||||||
if (entry->oid.inode)
|
|
||||||
{
|
{
|
||||||
printf(
|
clean_disk_entry *entry = (clean_disk_entry*)(data + blk + ioff);
|
||||||
|
if (entry->oid.inode)
|
||||||
|
{
|
||||||
|
printf(
|
||||||
#define ENTRY_FMT "{\"block\":%lu,\"pool\":%u,\"inode\":%lu,\"stripe\":%lu,\"version\":%lu,\"bitmap\":\""
|
#define ENTRY_FMT "{\"block\":%lu,\"pool\":%u,\"inode\":%lu,\"stripe\":%lu,\"version\":%lu,\"bitmap\":\""
|
||||||
(first ? ENTRY_FMT : (",\n" ENTRY_FMT)),
|
(first ? ENTRY_FMT : (",\n" ENTRY_FMT)),
|
||||||
#undef ENTRY_FMT
|
#undef ENTRY_FMT
|
||||||
block_num, INODE_POOL(entry->oid.inode), INODE_NO_POOL(entry->oid.inode),
|
block_num, INODE_POOL(entry->oid.inode), INODE_NO_POOL(entry->oid.inode),
|
||||||
entry->oid.stripe, entry->version
|
entry->oid.stripe, entry->version
|
||||||
);
|
);
|
||||||
first = false;
|
first = false;
|
||||||
for (uint64_t i = 0; i < clean_entry_bitmap_size; i++)
|
for (uint64_t i = 0; i < clean_entry_bitmap_size; i++)
|
||||||
{
|
{
|
||||||
printf("%02x", entry->bitmap[i]);
|
printf("%02x", entry->bitmap[i]);
|
||||||
|
}
|
||||||
|
printf("\",\"ext_bitmap\":\"");
|
||||||
|
for (uint64_t i = 0; i < clean_entry_bitmap_size; i++)
|
||||||
|
{
|
||||||
|
printf("%02x", entry->bitmap[clean_entry_bitmap_size + i]);
|
||||||
|
}
|
||||||
|
printf("\"}");
|
||||||
}
|
}
|
||||||
printf("\",\"ext_bitmap\":\"");
|
|
||||||
for (uint64_t i = 0; i < clean_entry_bitmap_size; i++)
|
|
||||||
{
|
|
||||||
printf("%02x", entry->bitmap[clean_entry_bitmap_size + i]);
|
|
||||||
}
|
|
||||||
printf("\"}");
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
printf("]}\n");
|
printf("\n]}\n");
|
||||||
}
|
}
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// Vitastor 0.4-0.5 - static array of clean_disk_entry_v0_t
|
// Vitastor 0.4-0.5 - static array of clean_disk_entry
|
||||||
uint64_t clean_entry_size = sizeof(clean_disk_entry_v0_t);
|
uint64_t clean_entry_size = sizeof(clean_disk_entry);
|
||||||
uint64_t block_num = 0;
|
uint64_t block_num = 0;
|
||||||
printf("{\"version\":\"0.5\",\"meta_block_size\":%u,\"entries\":[\n", self.meta_block_size);
|
printf("{\"version\":\"0.5\",\"meta_block_size\":%u,\"entries\":[\n", this->meta_block_size);
|
||||||
bool first = true;
|
bool first = true;
|
||||||
for (uint64_t meta_pos = 0; meta_pos < self.meta_len; meta_pos += self.meta_block_size)
|
while (this->meta_pos < this->meta_len)
|
||||||
{
|
{
|
||||||
for (uint64_t ioff = 0; ioff < self.meta_block_size-clean_entry_size; ioff += clean_entry_size, block_num++)
|
uint64_t read_len = buf_size < this->meta_len-this->meta_pos ? buf_size : this->meta_len-this->meta_pos;
|
||||||
|
read_blocking(this->fd, data, read_len);
|
||||||
|
this->meta_pos += read_len;
|
||||||
|
for (uint64_t blk = 0; blk < read_len; blk += this->meta_block_size)
|
||||||
{
|
{
|
||||||
clean_disk_entry_v0_t *entry = (clean_disk_entry_v0_t*)(data + meta_pos + ioff);
|
for (uint64_t ioff = 0; ioff < this->meta_block_size-clean_entry_size; ioff += clean_entry_size, block_num++)
|
||||||
if (entry->oid.inode)
|
|
||||||
{
|
{
|
||||||
printf(
|
clean_disk_entry *entry = (clean_disk_entry*)(data + blk + ioff);
|
||||||
|
if (entry->oid.inode)
|
||||||
|
{
|
||||||
|
printf(
|
||||||
#define ENTRY_FMT "{\"block\":%lu,\"pool\":%u,\"inode\":%lu,\"stripe\":%lu,\"version\":%lu}"
|
#define ENTRY_FMT "{\"block\":%lu,\"pool\":%u,\"inode\":%lu,\"stripe\":%lu,\"version\":%lu}"
|
||||||
(first ? ENTRY_FMT : (",\n" ENTRY_FMT)),
|
(first ? ENTRY_FMT : (",\n" ENTRY_FMT)),
|
||||||
#undef ENTRY_FMT
|
#undef ENTRY_FMT
|
||||||
block_num, INODE_POOL(entry->oid.inode), INODE_NO_POOL(entry->oid.inode),
|
block_num, INODE_POOL(entry->oid.inode), INODE_NO_POOL(entry->oid.inode),
|
||||||
entry->oid.stripe, entry->version
|
entry->oid.stripe, entry->version
|
||||||
);
|
);
|
||||||
first = false;
|
first = false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
printf("]}\n");
|
printf("\n]}\n");
|
||||||
}
|
}
|
||||||
free(data);
|
free(data);
|
||||||
|
close(this->fd);
|
||||||
|
return 0;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in New Issue