Fix degraded parity writes for EC N+K when K>1
Fixes possible `calc_rmw_parity_ec(): Assertion `bufs[i][curbuf[i]].buf' failed` errorrm-left-on-dead
parent
6e6f6ecbb0
commit
09a8864686
|
@ -478,17 +478,20 @@ void* calc_rmw(void *request_buf, osd_rmw_stripe_t *stripes, uint64_t *read_osd_
|
||||||
{
|
{
|
||||||
if (write_osd_set[role] != 0)
|
if (write_osd_set[role] != 0)
|
||||||
{
|
{
|
||||||
write_parity = 1;
|
write_parity++;
|
||||||
if (write_osd_set[role] != read_osd_set[role])
|
if (write_osd_set[role] != read_osd_set[role])
|
||||||
{
|
{
|
||||||
start = 0;
|
start = 0;
|
||||||
end = chunk_size;
|
end = chunk_size;
|
||||||
for (int r2 = pg_minsize; r2 < role; r2++)
|
for (int r2 = pg_minsize; r2 < role; r2++)
|
||||||
|
{
|
||||||
|
if (write_osd_set[r2] != 0)
|
||||||
{
|
{
|
||||||
stripes[r2].write_start = start;
|
stripes[r2].write_start = start;
|
||||||
stripes[r2].write_end = end;
|
stripes[r2].write_end = end;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
stripes[role].write_start = start;
|
stripes[role].write_start = start;
|
||||||
stripes[role].write_end = end;
|
stripes[role].write_end = end;
|
||||||
}
|
}
|
||||||
|
@ -555,7 +558,7 @@ void* calc_rmw(void *request_buf, osd_rmw_stripe_t *stripes, uint64_t *read_osd_
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Allocate read buffers
|
// Allocate read buffers
|
||||||
void *rmw_buf = alloc_read_buffer(stripes, pg_size, (write_parity ? pg_size-pg_minsize : 0) * (end - start));
|
void *rmw_buf = alloc_read_buffer(stripes, pg_size, write_parity * (end - start));
|
||||||
// Position write buffers
|
// Position write buffers
|
||||||
uint64_t buf_pos = 0, in_pos = 0;
|
uint64_t buf_pos = 0, in_pos = 0;
|
||||||
for (int role = 0; role < pg_size; role++)
|
for (int role = 0; role < pg_size; role++)
|
||||||
|
@ -804,13 +807,11 @@ void calc_rmw_parity_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize,
|
||||||
calc_rmw_parity_copy_mod(stripes, pg_size, pg_minsize, read_osd_set, write_osd_set, chunk_size, bitmap_granularity, start, end);
|
calc_rmw_parity_copy_mod(stripes, pg_size, pg_minsize, read_osd_set, write_osd_set, chunk_size, bitmap_granularity, start, end);
|
||||||
if (end != 0)
|
if (end != 0)
|
||||||
{
|
{
|
||||||
int i;
|
int write_parity = 0;
|
||||||
for (i = pg_minsize; i < pg_size; i++)
|
for (int i = pg_minsize; i < pg_size; i++)
|
||||||
{
|
|
||||||
if (write_osd_set[i] != 0)
|
if (write_osd_set[i] != 0)
|
||||||
break;
|
write_parity++;
|
||||||
}
|
if (write_parity > 0)
|
||||||
if (i < pg_size)
|
|
||||||
{
|
{
|
||||||
// Calculate new coding chunks
|
// Calculate new coding chunks
|
||||||
buf_len_t bufs[pg_size][3];
|
buf_len_t bufs[pg_size][3];
|
||||||
|
@ -829,15 +830,20 @@ void calc_rmw_parity_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize,
|
||||||
positions[i] = start;
|
positions[i] = start;
|
||||||
}
|
}
|
||||||
for (int i = pg_minsize; i < pg_size; i++)
|
for (int i = pg_minsize; i < pg_size; i++)
|
||||||
|
{
|
||||||
|
if (write_osd_set[i] != 0)
|
||||||
{
|
{
|
||||||
bufs[i][nbuf[i]++] = { .buf = stripes[i].write_buf, .len = end-start };
|
bufs[i][nbuf[i]++] = { .buf = stripes[i].write_buf, .len = end-start };
|
||||||
positions[i] = start;
|
positions[i] = start;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
uint32_t pos = start;
|
uint32_t pos = start;
|
||||||
while (pos < end)
|
while (pos < end)
|
||||||
{
|
{
|
||||||
uint32_t next_end = end;
|
uint32_t next_end = end;
|
||||||
for (int i = 0; i < pg_size; i++)
|
for (int i = 0; i < pg_size; i++)
|
||||||
|
{
|
||||||
|
if (i < pg_minsize || write_osd_set[i] != 0)
|
||||||
{
|
{
|
||||||
assert(curbuf[i] < nbuf[i]);
|
assert(curbuf[i] < nbuf[i]);
|
||||||
assert(bufs[i][curbuf[i]].buf);
|
assert(bufs[i][curbuf[i]].buf);
|
||||||
|
@ -846,8 +852,11 @@ void calc_rmw_parity_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize,
|
||||||
if (next_end > this_end)
|
if (next_end > this_end)
|
||||||
next_end = this_end;
|
next_end = this_end;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
assert(next_end > pos);
|
assert(next_end > pos);
|
||||||
for (int i = 0; i < pg_size; i++)
|
for (int i = 0; i < pg_size; i++)
|
||||||
|
{
|
||||||
|
if (i < pg_minsize || write_osd_set[i] != 0)
|
||||||
{
|
{
|
||||||
uint32_t this_end = bufs[i][curbuf[i]].len + positions[i];
|
uint32_t this_end = bufs[i][curbuf[i]].len + positions[i];
|
||||||
if (next_end >= this_end)
|
if (next_end >= this_end)
|
||||||
|
@ -856,31 +865,35 @@ void calc_rmw_parity_ec(osd_rmw_stripe_t *stripes, int pg_size, int pg_minsize,
|
||||||
curbuf[i]++;
|
curbuf[i]++;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
#ifdef WITH_ISAL
|
#ifdef WITH_ISAL
|
||||||
ec_encode_data(
|
ec_encode_data(
|
||||||
next_end-pos, pg_minsize, pg_size-pg_minsize, matrix->isal_data,
|
next_end-pos, pg_minsize, write_parity, matrix->isal_data,
|
||||||
(uint8_t**)data_ptrs, (uint8_t**)data_ptrs+pg_minsize
|
(uint8_t**)data_ptrs, (uint8_t**)data_ptrs+pg_minsize
|
||||||
);
|
);
|
||||||
#else
|
#else
|
||||||
jerasure_matrix_encode(
|
jerasure_matrix_encode(
|
||||||
pg_minsize, pg_size-pg_minsize, OSD_JERASURE_W, matrix->je_data,
|
pg_minsize, write_parity, OSD_JERASURE_W, matrix->je_data,
|
||||||
(char**)data_ptrs, (char**)data_ptrs+pg_minsize, next_end-pos
|
(char**)data_ptrs, (char**)data_ptrs+pg_minsize, next_end-pos
|
||||||
);
|
);
|
||||||
#endif
|
#endif
|
||||||
pos = next_end;
|
pos = next_end;
|
||||||
}
|
}
|
||||||
for (int i = 0; i < pg_size; i++)
|
for (int i = 0; i < pg_size; i++)
|
||||||
|
{
|
||||||
|
if (i < pg_minsize || write_osd_set[i] != 0)
|
||||||
{
|
{
|
||||||
data_ptrs[i] = stripes[i].bmp_buf;
|
data_ptrs[i] = stripes[i].bmp_buf;
|
||||||
}
|
}
|
||||||
|
}
|
||||||
#ifdef WITH_ISAL
|
#ifdef WITH_ISAL
|
||||||
ec_encode_data(
|
ec_encode_data(
|
||||||
bitmap_size, pg_minsize, pg_size-pg_minsize, matrix->isal_data,
|
bitmap_size, pg_minsize, write_parity, matrix->isal_data,
|
||||||
(uint8_t**)data_ptrs, (uint8_t**)data_ptrs+pg_minsize
|
(uint8_t**)data_ptrs, (uint8_t**)data_ptrs+pg_minsize
|
||||||
);
|
);
|
||||||
#else
|
#else
|
||||||
jerasure_matrix_encode(
|
jerasure_matrix_encode(
|
||||||
pg_minsize, pg_size-pg_minsize, OSD_JERASURE_W, matrix->je_data,
|
pg_minsize, write_parity, OSD_JERASURE_W, matrix->je_data,
|
||||||
(char**)data_ptrs, (char**)data_ptrs+pg_minsize, bitmap_size
|
(char**)data_ptrs, (char**)data_ptrs+pg_minsize, bitmap_size
|
||||||
);
|
);
|
||||||
#endif
|
#endif
|
||||||
|
|
|
@ -20,6 +20,7 @@ void test11();
|
||||||
void test12();
|
void test12();
|
||||||
void test13();
|
void test13();
|
||||||
void test14();
|
void test14();
|
||||||
|
void test15();
|
||||||
|
|
||||||
int main(int narg, char *args[])
|
int main(int narg, char *args[])
|
||||||
{
|
{
|
||||||
|
@ -47,6 +48,8 @@ int main(int narg, char *args[])
|
||||||
test13();
|
test13();
|
||||||
// Test 14
|
// Test 14
|
||||||
test14();
|
test14();
|
||||||
|
// Test 15
|
||||||
|
test15();
|
||||||
// End
|
// End
|
||||||
printf("all ok\n");
|
printf("all ok\n");
|
||||||
return 0;
|
return 0;
|
||||||
|
@ -706,7 +709,7 @@ void test13()
|
||||||
|
|
||||||
/***
|
/***
|
||||||
|
|
||||||
13. basic jerasure 2+1 test
|
14. basic jerasure 2+1 test
|
||||||
calc_rmw(offset=128K-4K, len=8K, osd_set=[1,2,0], write_set=[1,2,3])
|
calc_rmw(offset=128K-4K, len=8K, osd_set=[1,2,0], write_set=[1,2,3])
|
||||||
= {
|
= {
|
||||||
read: [ [ 0, 128K ], [ 0, 128K ], [ 0, 0 ] ],
|
read: [ [ 0, 128K ], [ 0, 128K ], [ 0, 0 ] ],
|
||||||
|
@ -727,13 +730,13 @@ void test14()
|
||||||
osd_num_t write_osd_set[3] = { 1, 2, 3 };
|
osd_num_t write_osd_set[3] = { 1, 2, 3 };
|
||||||
osd_rmw_stripe_t stripes[3] = {};
|
osd_rmw_stripe_t stripes[3] = {};
|
||||||
unsigned bitmaps[3] = { 0 };
|
unsigned bitmaps[3] = { 0 };
|
||||||
// Test 13.0
|
// Test 14.0
|
||||||
void *write_buf = malloc_or_die(8192);
|
void *write_buf = malloc_or_die(8192);
|
||||||
split_stripes(2, 128*1024, 128*1024-4096, 8192, stripes);
|
split_stripes(2, 128*1024, 128*1024-4096, 8192, stripes);
|
||||||
assert(stripes[0].req_start == 128*1024-4096 && stripes[0].req_end == 128*1024);
|
assert(stripes[0].req_start == 128*1024-4096 && stripes[0].req_end == 128*1024);
|
||||||
assert(stripes[1].req_start == 0 && stripes[1].req_end == 4096);
|
assert(stripes[1].req_start == 0 && stripes[1].req_end == 4096);
|
||||||
assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
|
assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
|
||||||
// Test 13.1
|
// Test 14.1
|
||||||
void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 3, 2, 3, write_osd_set, 128*1024, bmp);
|
void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 3, 2, 3, write_osd_set, 128*1024, bmp);
|
||||||
for (int i = 0; i < 3; i++)
|
for (int i = 0; i < 3; i++)
|
||||||
stripes[i].bmp_buf = bitmaps+i;
|
stripes[i].bmp_buf = bitmaps+i;
|
||||||
|
@ -750,7 +753,7 @@ void test14()
|
||||||
assert(stripes[0].write_buf == write_buf);
|
assert(stripes[0].write_buf == write_buf);
|
||||||
assert(stripes[1].write_buf == (uint8_t*)write_buf+4096);
|
assert(stripes[1].write_buf == (uint8_t*)write_buf+4096);
|
||||||
assert(stripes[2].write_buf == rmw_buf);
|
assert(stripes[2].write_buf == rmw_buf);
|
||||||
// Test 13.2 - encode
|
// Test 14.2 - encode
|
||||||
set_pattern(write_buf, 8192, PATTERN3);
|
set_pattern(write_buf, 8192, PATTERN3);
|
||||||
set_pattern(stripes[0].read_buf, 128*1024-4096, PATTERN1);
|
set_pattern(stripes[0].read_buf, 128*1024-4096, PATTERN1);
|
||||||
set_pattern(stripes[1].read_buf, 128*1024-4096, PATTERN2);
|
set_pattern(stripes[1].read_buf, 128*1024-4096, PATTERN2);
|
||||||
|
@ -767,7 +770,7 @@ void test14()
|
||||||
assert(stripes[0].write_buf == write_buf);
|
assert(stripes[0].write_buf == write_buf);
|
||||||
assert(stripes[1].write_buf == (uint8_t*)write_buf+4096);
|
assert(stripes[1].write_buf == (uint8_t*)write_buf+4096);
|
||||||
assert(stripes[2].write_buf == rmw_buf);
|
assert(stripes[2].write_buf == rmw_buf);
|
||||||
// Test 13.3 - decode and verify
|
// Test 14.3 - decode and verify
|
||||||
osd_num_t read_osd_set[4] = { 0, 2, 3 };
|
osd_num_t read_osd_set[4] = { 0, 2, 3 };
|
||||||
memset(stripes, 0, sizeof(stripes));
|
memset(stripes, 0, sizeof(stripes));
|
||||||
split_stripes(2, 128*1024, 0, 128*1024, stripes);
|
split_stripes(2, 128*1024, 0, 128*1024, stripes);
|
||||||
|
@ -802,3 +805,74 @@ void test14()
|
||||||
free(write_buf);
|
free(write_buf);
|
||||||
use_ec(3, 2, false);
|
use_ec(3, 2, false);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/***
|
||||||
|
|
||||||
|
15. EC 2+2 partial overwrite with 1 missing stripe
|
||||||
|
calc_rmw(offset=64K+28K, len=4K, osd_set=[1,2,3,0], write_set=[1,2,3,0])
|
||||||
|
= {
|
||||||
|
read: [ [ 28K, 32K ], [ 0, 0 ], [ 0, 0 ], [ 0, 0 ] ],
|
||||||
|
write: [ [ 0, 0 ], [ 28K, 32K ], [ 28K, 32K ], [ 0, 0 ] ],
|
||||||
|
input buffer: [ write1 ],
|
||||||
|
rmw buffer: [ write2, read0 ],
|
||||||
|
}
|
||||||
|
|
||||||
|
***/
|
||||||
|
|
||||||
|
void test15()
|
||||||
|
{
|
||||||
|
const int bmp = 64*1024 / 4096 / 8;
|
||||||
|
use_ec(4, 2, true);
|
||||||
|
osd_num_t osd_set[4] = { 1, 2, 3, 0 };
|
||||||
|
osd_num_t write_osd_set[4] = { 1, 2, 3, 0 };
|
||||||
|
osd_rmw_stripe_t stripes[4] = {};
|
||||||
|
unsigned bitmaps[4] = { 0 };
|
||||||
|
// Test 15.0
|
||||||
|
void *write_buf = malloc_or_die(4096);
|
||||||
|
split_stripes(2, 64*1024, (64+28)*1024, 4096, stripes);
|
||||||
|
assert(stripes[0].req_start == 0 && stripes[0].req_end == 0);
|
||||||
|
assert(stripes[1].req_start == 28*1024 && stripes[1].req_end == 32*1024);
|
||||||
|
assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
|
||||||
|
assert(stripes[3].req_start == 0 && stripes[3].req_end == 0);
|
||||||
|
// Test 15.1
|
||||||
|
void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 4, 2, 3, write_osd_set, 64*1024, bmp);
|
||||||
|
for (int i = 0; i < 4; i++)
|
||||||
|
stripes[i].bmp_buf = bitmaps+i;
|
||||||
|
assert(rmw_buf);
|
||||||
|
assert(stripes[0].read_start == 28*1024 && stripes[0].read_end == 32*1024);
|
||||||
|
assert(stripes[1].read_start == 0 && stripes[1].read_end == 0);
|
||||||
|
assert(stripes[2].read_start == 0 && stripes[2].read_end == 0);
|
||||||
|
assert(stripes[3].read_start == 0 && stripes[3].read_end == 0);
|
||||||
|
assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
|
||||||
|
assert(stripes[1].write_start == 28*1024 && stripes[1].write_end == 32*1024);
|
||||||
|
assert(stripes[2].write_start == 28*1024 && stripes[2].write_end == 32*1024);
|
||||||
|
assert(stripes[3].write_start == 0 && stripes[3].write_end == 0);
|
||||||
|
assert(stripes[0].read_buf == (uint8_t*)rmw_buf+4*1024);
|
||||||
|
assert(stripes[1].read_buf == NULL);
|
||||||
|
assert(stripes[2].read_buf == NULL);
|
||||||
|
assert(stripes[3].read_buf == NULL);
|
||||||
|
assert(stripes[0].write_buf == NULL);
|
||||||
|
assert(stripes[1].write_buf == (uint8_t*)write_buf);
|
||||||
|
assert(stripes[2].write_buf == rmw_buf);
|
||||||
|
assert(stripes[3].write_buf == NULL);
|
||||||
|
// Test 15.2 - encode
|
||||||
|
set_pattern(write_buf, 4*1024, PATTERN1);
|
||||||
|
set_pattern(stripes[0].read_buf, 4*1024, PATTERN2);
|
||||||
|
memset(stripes[0].bmp_buf, 0, bmp);
|
||||||
|
memset(stripes[1].bmp_buf, 0, bmp);
|
||||||
|
calc_rmw_parity_ec(stripes, 4, 2, osd_set, write_osd_set, 64*1024, bmp);
|
||||||
|
assert(*(uint32_t*)stripes[2].bmp_buf == 0x80);
|
||||||
|
assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
|
||||||
|
assert(stripes[1].write_start == 28*1024 && stripes[1].write_end == 32*1024);
|
||||||
|
assert(stripes[2].write_start == 28*1024 && stripes[2].write_end == 32*1024);
|
||||||
|
assert(stripes[3].write_start == 0 && stripes[3].write_end == 0);
|
||||||
|
assert(stripes[0].write_buf == NULL);
|
||||||
|
assert(stripes[1].write_buf == (uint8_t*)write_buf);
|
||||||
|
assert(stripes[2].write_buf == rmw_buf);
|
||||||
|
assert(stripes[3].write_buf == NULL);
|
||||||
|
check_pattern(stripes[2].write_buf, 4*1024, PATTERN1^PATTERN2); // first parity is always xor :)
|
||||||
|
// Done
|
||||||
|
free(rmw_buf);
|
||||||
|
free(write_buf);
|
||||||
|
use_ec(3, 2, false);
|
||||||
|
}
|
||||||
|
|
Loading…
Reference in New Issue