Simplified distributed block storage with strong consistency, like in Ceph
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

805 lines
33 KiB

  1. // Copyright (c) Vitaliy Filippov, 2019+
  2. // License: VNPL-1.0 (see README.md for details)
  3. #define RMW_DEBUG
  4. #include <string.h>
  5. #include "osd_rmw.cpp"
  6. #include "test_pattern.h"
  7. void dump_stripes(osd_rmw_stripe_t *stripes, int pg_size);
  8. void test1();
  9. void test4();
  10. void test5();
  11. void test6();
  12. void test7();
  13. void test8();
  14. void test9();
  15. void test10();
  16. void test11();
  17. void test12();
  18. void test13();
  19. void test14();
  20. int main(int narg, char *args[])
  21. {
  22. // Test 1
  23. test1();
  24. // Test 4
  25. test4();
  26. // Test 5
  27. test5();
  28. // Test 6
  29. test6();
  30. // Test 7
  31. test7();
  32. // Test 8
  33. test8();
  34. // Test 9
  35. test9();
  36. // Test 10
  37. test10();
  38. // Test 11
  39. test11();
  40. // Test 12
  41. test12();
  42. // Test 13
  43. test13();
  44. // Test 14
  45. test14();
  46. // End
  47. printf("all ok\n");
  48. return 0;
  49. }
  50. void dump_stripes(osd_rmw_stripe_t *stripes, int pg_size)
  51. {
  52. printf("request");
  53. for (int i = 0; i < pg_size; i++)
  54. {
  55. printf(" {%uK-%uK}", stripes[i].req_start/1024, stripes[i].req_end/1024);
  56. }
  57. printf("\n");
  58. printf("read");
  59. for (int i = 0; i < pg_size; i++)
  60. {
  61. printf(" {%uK-%uK}", stripes[i].read_start/1024, stripes[i].read_end/1024);
  62. }
  63. printf("\n");
  64. printf("write");
  65. for (int i = 0; i < pg_size; i++)
  66. {
  67. printf(" {%uK-%uK}", stripes[i].write_start/1024, stripes[i].write_end/1024);
  68. }
  69. printf("\n");
  70. }
  71. /***
  72. 1. split(offset=128K-4K, len=8K)
  73. = [ [ 128K-4K, 128K ], [ 0, 4K ], [ 0, 0 ] ]
  74. read(offset=128K-4K, len=8K, osd_set=[1,0,3])
  75. = { read: [ [ 0, 128K ], [ 0, 4K ], [ 0, 4K ] ] }
  76. cover_read(0, 128K, { req: [ 128K-4K, 4K ] })
  77. = { read: [ 0, 128K-4K ] }
  78. ***/
  79. void test1()
  80. {
  81. osd_num_t osd_set[3] = { 1, 0, 3 };
  82. osd_rmw_stripe_t stripes[3] = { 0 };
  83. // Test 1.1
  84. split_stripes(2, 128*1024, 128*1024-4096, 8192, stripes);
  85. assert(stripes[0].req_start == 128*1024-4096 && stripes[0].req_end == 128*1024);
  86. assert(stripes[1].req_start == 0 && stripes[1].req_end == 4096);
  87. assert(stripes[2].req_end == 0);
  88. // Test 1.2
  89. for (int i = 0; i < 3; i++)
  90. {
  91. stripes[i].read_start = stripes[i].req_start;
  92. stripes[i].read_end = stripes[i].req_end;
  93. }
  94. assert(extend_missing_stripes(stripes, osd_set, 2, 3) == 0);
  95. assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024);
  96. assert(stripes[2].read_start == 0 && stripes[2].read_end == 4096);
  97. // Test 1.3
  98. stripes[0] = (osd_rmw_stripe_t){ .req_start = 128*1024-4096, .req_end = 128*1024 };
  99. cover_read(0, 128*1024, stripes[0]);
  100. assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024-4096);
  101. }
  102. /***
  103. 4. write(offset=128K-4K, len=8K, osd_set=[1,0,3])
  104. = {
  105. read: [ [ 0, 128K ], [ 4K, 128K ], [ 4K, 128K ] ],
  106. write: [ [ 128K-4K, 128K ], [ 0, 4K ], [ 0, 128K ] ],
  107. input buffer: [ write0, write1 ],
  108. rmw buffer: [ write2, read0, read1, read2 ],
  109. }
  110. + check write2 buffer
  111. ***/
  112. void test4()
  113. {
  114. const uint32_t bmp = 4;
  115. unsigned bitmaps[3] = { 0 };
  116. osd_num_t osd_set[3] = { 1, 0, 3 };
  117. osd_rmw_stripe_t stripes[3] = { 0 };
  118. // Test 4.1
  119. split_stripes(2, 128*1024, 128*1024-4096, 8192, stripes);
  120. for (int i = 0; i < 3; i++)
  121. stripes[i].bmp_buf = bitmaps+i;
  122. void* write_buf = malloc(8192);
  123. void* rmw_buf = calc_rmw(write_buf, stripes, osd_set, 3, 2, 2, osd_set, 128*1024, bmp);
  124. assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024);
  125. assert(stripes[1].read_start == 4096 && stripes[1].read_end == 128*1024);
  126. assert(stripes[2].read_start == 4096 && stripes[2].read_end == 128*1024);
  127. assert(stripes[0].write_start == 128*1024-4096 && stripes[0].write_end == 128*1024);
  128. assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);
  129. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  130. assert(stripes[0].read_buf == rmw_buf+128*1024);
  131. assert(stripes[1].read_buf == rmw_buf+128*1024*2);
  132. assert(stripes[2].read_buf == rmw_buf+128*1024*3-4096);
  133. assert(stripes[0].write_buf == write_buf);
  134. assert(stripes[1].write_buf == write_buf+4096);
  135. assert(stripes[2].write_buf == rmw_buf);
  136. // Test 4.2
  137. set_pattern(write_buf, 8192, PATTERN0);
  138. set_pattern(stripes[0].read_buf, 128*1024, PATTERN1); // old data
  139. set_pattern(stripes[1].read_buf, 128*1024-4096, UINT64_MAX); // didn't read it, it's missing
  140. set_pattern(stripes[2].read_buf, 128*1024-4096, 0); // old parity = 0
  141. memset(stripes[0].bmp_buf, 0, bmp);
  142. memset(stripes[1].bmp_buf, 0, bmp);
  143. memset(stripes[2].bmp_buf, 0, bmp);
  144. calc_rmw_parity_xor(stripes, 3, osd_set, osd_set, 128*1024, bmp);
  145. assert(*(uint32_t*)stripes[0].bmp_buf == 0x80000000);
  146. assert(*(uint32_t*)stripes[1].bmp_buf == 0x00000001);
  147. assert(*(uint32_t*)stripes[2].bmp_buf == 0x80000001); // XOR
  148. check_pattern(stripes[2].write_buf, 4096, PATTERN0^PATTERN1); // new parity
  149. check_pattern(stripes[2].write_buf+4096, 128*1024-4096*2, 0); // new parity
  150. check_pattern(stripes[2].write_buf+128*1024-4096, 4096, PATTERN0^PATTERN1); // new parity
  151. free(rmw_buf);
  152. free(write_buf);
  153. }
  154. /***
  155. 5. write(offset=0, len=128K+64K, osd_set=[1,0,3])
  156. = {
  157. req: [ [ 0, 128K ], [ 0, 64K ], [ 0, 0 ] ],
  158. read: [ [ 64K, 128K ], [ 64K, 128K ], [ 64K, 128K ] ],
  159. write: [ [ 0, 128K ], [ 0, 64K ], [ 0, 128K ] ],
  160. input buffer: [ write0, write1 ],
  161. rmw buffer: [ write2, read0, read1, read2 ],
  162. }
  163. ***/
  164. void test5()
  165. {
  166. osd_num_t osd_set[3] = { 1, 0, 3 };
  167. osd_rmw_stripe_t stripes[3] = { 0 };
  168. // Test 5.1
  169. split_stripes(2, 128*1024, 0, 64*1024*3, stripes);
  170. assert(stripes[0].req_start == 0 && stripes[0].req_end == 128*1024);
  171. assert(stripes[1].req_start == 0 && stripes[1].req_end == 64*1024);
  172. assert(stripes[2].req_end == 0);
  173. // Test 5.2
  174. void *write_buf = malloc(64*1024*3);
  175. void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 3, 2, 2, osd_set, 128*1024, 0);
  176. assert(stripes[0].read_start == 64*1024 && stripes[0].read_end == 128*1024);
  177. assert(stripes[1].read_start == 64*1024 && stripes[1].read_end == 128*1024);
  178. assert(stripes[2].read_start == 64*1024 && stripes[2].read_end == 128*1024);
  179. assert(stripes[0].write_start == 0 && stripes[0].write_end == 128*1024);
  180. assert(stripes[1].write_start == 0 && stripes[1].write_end == 64*1024);
  181. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  182. assert(stripes[0].read_buf == rmw_buf+128*1024);
  183. assert(stripes[1].read_buf == rmw_buf+64*3*1024);
  184. assert(stripes[2].read_buf == rmw_buf+64*4*1024);
  185. assert(stripes[0].write_buf == write_buf);
  186. assert(stripes[1].write_buf == write_buf+128*1024);
  187. assert(stripes[2].write_buf == rmw_buf);
  188. free(rmw_buf);
  189. free(write_buf);
  190. }
  191. /***
  192. 6. write(offset=0, len=128K+64K, osd_set=[1,2,3])
  193. = {
  194. req: [ [ 0, 128K ], [ 0, 64K ], [ 0, 0 ] ],
  195. read: [ [ 0, 0 ], [ 64K, 128K ], [ 0, 0 ] ],
  196. write: [ [ 0, 128K ], [ 0, 64K ], [ 0, 128K ] ],
  197. input buffer: [ write0, write1 ],
  198. rmw buffer: [ write2, read1 ],
  199. }
  200. ***/
  201. void test6()
  202. {
  203. osd_num_t osd_set[3] = { 1, 2, 3 };
  204. osd_rmw_stripe_t stripes[3] = { 0 };
  205. // Test 6.1
  206. split_stripes(2, 128*1024, 0, 64*1024*3, stripes);
  207. void *write_buf = malloc(64*1024*3);
  208. void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 3, 2, 3, osd_set, 128*1024, 0);
  209. assert(stripes[0].read_end == 0);
  210. assert(stripes[1].read_start == 64*1024 && stripes[1].read_end == 128*1024);
  211. assert(stripes[2].read_end == 0);
  212. assert(stripes[0].write_start == 0 && stripes[0].write_end == 128*1024);
  213. assert(stripes[1].write_start == 0 && stripes[1].write_end == 64*1024);
  214. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  215. assert(stripes[0].read_buf == 0);
  216. assert(stripes[1].read_buf == rmw_buf+128*1024);
  217. assert(stripes[2].read_buf == 0);
  218. assert(stripes[0].write_buf == write_buf);
  219. assert(stripes[1].write_buf == write_buf+128*1024);
  220. assert(stripes[2].write_buf == rmw_buf);
  221. free(rmw_buf);
  222. free(write_buf);
  223. }
  224. /***
  225. 7. calc_rmw(offset=128K-4K, len=8K, osd_set=[1,0,3], write_set=[1,2,3])
  226. = {
  227. read: [ [ 0, 128K ], [ 0, 128K ], [ 0, 128K ] ],
  228. write: [ [ 128K-4K, 128K ], [ 0, 4K ], [ 0, 128K ] ],
  229. input buffer: [ write0, write1 ],
  230. rmw buffer: [ write2, read0, read1, read2 ],
  231. }
  232. then, after calc_rmw_parity_xor(): {
  233. write: [ [ 128K-4K, 128K ], [ 0, 128K ], [ 0, 128K ] ],
  234. write1==read1,
  235. }
  236. + check write1 buffer
  237. + check write2 buffer
  238. ***/
  239. void test7()
  240. {
  241. osd_num_t osd_set[3] = { 1, 0, 3 };
  242. osd_num_t write_osd_set[3] = { 1, 2, 3 };
  243. osd_rmw_stripe_t stripes[3] = { 0 };
  244. // Test 7.1
  245. split_stripes(2, 128*1024, 128*1024-4096, 8192, stripes);
  246. void *write_buf = malloc(8192);
  247. void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 3, 2, 2, write_osd_set, 128*1024, 0);
  248. assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024);
  249. assert(stripes[1].read_start == 0 && stripes[1].read_end == 128*1024);
  250. assert(stripes[2].read_start == 0 && stripes[2].read_end == 128*1024);
  251. assert(stripes[0].write_start == 128*1024-4096 && stripes[0].write_end == 128*1024);
  252. assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);
  253. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  254. assert(stripes[0].read_buf == rmw_buf+128*1024);
  255. assert(stripes[1].read_buf == rmw_buf+128*1024*2);
  256. assert(stripes[2].read_buf == rmw_buf+128*1024*3);
  257. assert(stripes[0].write_buf == write_buf);
  258. assert(stripes[1].write_buf == write_buf+4096);
  259. assert(stripes[2].write_buf == rmw_buf);
  260. // Test 7.2
  261. set_pattern(write_buf, 8192, PATTERN0);
  262. set_pattern(stripes[0].read_buf, 128*1024, PATTERN1); // old data
  263. set_pattern(stripes[1].read_buf, 128*1024, UINT64_MAX); // didn't read it, it's missing
  264. set_pattern(stripes[2].read_buf, 128*1024, 0); // old parity = 0
  265. calc_rmw_parity_xor(stripes, 3, osd_set, write_osd_set, 128*1024, 0);
  266. assert(stripes[0].write_start == 128*1024-4096 && stripes[0].write_end == 128*1024);
  267. assert(stripes[1].write_start == 0 && stripes[1].write_end == 128*1024);
  268. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  269. assert(stripes[1].write_buf == stripes[1].read_buf);
  270. check_pattern(stripes[1].write_buf, 4096, PATTERN0);
  271. check_pattern(stripes[1].write_buf+4096, 128*1024-4096, PATTERN1);
  272. check_pattern(stripes[2].write_buf, 4096, PATTERN0^PATTERN1); // new parity
  273. check_pattern(stripes[2].write_buf+4096, 128*1024-4096*2, 0); // new parity
  274. check_pattern(stripes[2].write_buf+128*1024-4096, 4096, PATTERN0^PATTERN1); // new parity
  275. free(rmw_buf);
  276. free(write_buf);
  277. }
  278. /***
  279. 8. calc_rmw(offset=0, len=128K+4K, osd_set=[0,2,3], write_set=[1,2,3])
  280. = {
  281. read: [ [ 0, 0 ], [ 4K, 128K ], [ 0, 0 ] ],
  282. write: [ [ 0, 128K ], [ 0, 4K ], [ 0, 128K ] ],
  283. input buffer: [ write0, write1 ],
  284. rmw buffer: [ write2, read1 ],
  285. }
  286. + check write2 buffer
  287. ***/
  288. void test8()
  289. {
  290. osd_num_t osd_set[3] = { 0, 2, 3 };
  291. osd_num_t write_osd_set[3] = { 1, 2, 3 };
  292. osd_rmw_stripe_t stripes[3] = { 0 };
  293. // Test 8.1
  294. split_stripes(2, 128*1024, 0, 128*1024+4096, stripes);
  295. void *write_buf = malloc(128*1024+4096);
  296. void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 3, 2, 2, write_osd_set, 128*1024, 0);
  297. assert(stripes[0].read_start == 0 && stripes[0].read_end == 0);
  298. assert(stripes[1].read_start == 4096 && stripes[1].read_end == 128*1024);
  299. assert(stripes[2].read_start == 0 && stripes[2].read_end == 0);
  300. assert(stripes[0].write_start == 0 && stripes[0].write_end == 128*1024);
  301. assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);
  302. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  303. assert(stripes[0].read_buf == NULL);
  304. assert(stripes[1].read_buf == rmw_buf+128*1024);
  305. assert(stripes[2].read_buf == NULL);
  306. assert(stripes[0].write_buf == write_buf);
  307. assert(stripes[1].write_buf == write_buf+128*1024);
  308. assert(stripes[2].write_buf == rmw_buf);
  309. // Test 8.2
  310. set_pattern(write_buf, 128*1024+4096, PATTERN0);
  311. set_pattern(stripes[1].read_buf, 128*1024-4096, PATTERN1);
  312. calc_rmw_parity_xor(stripes, 3, osd_set, write_osd_set, 128*1024, 0);
  313. assert(stripes[0].write_start == 0 && stripes[0].write_end == 128*1024); // recheck again
  314. assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096); // recheck again
  315. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024); // recheck again
  316. assert(stripes[0].write_buf == write_buf); // recheck again
  317. assert(stripes[1].write_buf == write_buf+128*1024); // recheck again
  318. assert(stripes[2].write_buf == rmw_buf); // recheck again
  319. check_pattern(stripes[2].write_buf, 4096, 0); // new parity
  320. check_pattern(stripes[2].write_buf+4096, 128*1024-4096, PATTERN0^PATTERN1); // new parity
  321. free(rmw_buf);
  322. free(write_buf);
  323. }
  324. /***
  325. 9. object recovery case:
  326. calc_rmw(offset=0, len=0, read_osd_set=[0,2,3], write_osd_set=[1,2,3])
  327. = {
  328. read: [ [ 0, 128K ], [ 0, 128K ], [ 0, 128K ] ],
  329. write: [ [ 0, 0 ], [ 0, 0 ], [ 0, 0 ] ],
  330. input buffer: NULL,
  331. rmw buffer: [ read0, read1, read2 ],
  332. }
  333. then, after calc_rmw_parity_xor(): {
  334. write: [ [ 0, 128K ], [ 0, 0 ], [ 0, 0 ] ],
  335. write0==read0,
  336. }
  337. + check write0 buffer
  338. ***/
  339. void test9()
  340. {
  341. osd_num_t osd_set[3] = { 0, 2, 3 };
  342. osd_num_t write_osd_set[3] = { 1, 2, 3 };
  343. osd_rmw_stripe_t stripes[3] = { 0 };
  344. // Test 9.0
  345. split_stripes(2, 128*1024, 64*1024, 0, stripes);
  346. assert(stripes[0].req_start == 0 && stripes[0].req_end == 0);
  347. assert(stripes[1].req_start == 0 && stripes[1].req_end == 0);
  348. assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
  349. // Test 9.1
  350. void *write_buf = NULL;
  351. void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 3, 2, 3, write_osd_set, 128*1024, 0);
  352. assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024);
  353. assert(stripes[1].read_start == 0 && stripes[1].read_end == 128*1024);
  354. assert(stripes[2].read_start == 0 && stripes[2].read_end == 128*1024);
  355. assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
  356. assert(stripes[1].write_start == 0 && stripes[1].write_end == 0);
  357. assert(stripes[2].write_start == 0 && stripes[2].write_end == 0);
  358. assert(stripes[0].read_buf == rmw_buf);
  359. assert(stripes[1].read_buf == rmw_buf+128*1024);
  360. assert(stripes[2].read_buf == rmw_buf+128*1024*2);
  361. assert(stripes[0].write_buf == NULL);
  362. assert(stripes[1].write_buf == NULL);
  363. assert(stripes[2].write_buf == NULL);
  364. // Test 9.2
  365. set_pattern(stripes[1].read_buf, 128*1024, 0);
  366. set_pattern(stripes[2].read_buf, 128*1024, PATTERN1);
  367. calc_rmw_parity_xor(stripes, 3, osd_set, write_osd_set, 128*1024, 0);
  368. assert(stripes[0].write_start == 0 && stripes[0].write_end == 128*1024);
  369. assert(stripes[1].write_start == 0 && stripes[1].write_end == 0);
  370. assert(stripes[2].write_start == 0 && stripes[2].write_end == 0);
  371. assert(stripes[0].write_buf == rmw_buf);
  372. assert(stripes[1].write_buf == NULL);
  373. assert(stripes[2].write_buf == NULL);
  374. check_pattern(stripes[0].read_buf, 128*1024, PATTERN1);
  375. check_pattern(stripes[0].write_buf, 128*1024, PATTERN1);
  376. free(rmw_buf);
  377. }
  378. /***
  379. 10. full overwrite/recovery case:
  380. calc_rmw(offset=0, len=256K, read_osd_set=[1,0,0], write_osd_set=[1,2,3])
  381. = {
  382. read: [ [ 0, 0 ], [ 0, 0 ], [ 0, 0 ] ],
  383. write: [ [ 0, 128K ], [ 0, 128K ], [ 0, 128K ] ],
  384. input buffer: [ write0, write1 ],
  385. rmw buffer: [ write2 ],
  386. }
  387. then, after calc_rmw_parity_xor(): all the same
  388. + check write2 buffer
  389. ***/
  390. void test10()
  391. {
  392. osd_num_t osd_set[3] = { 1, 0, 0 };
  393. osd_num_t write_osd_set[3] = { 1, 2, 3 };
  394. osd_rmw_stripe_t stripes[3] = { 0 };
  395. // Test 10.0
  396. split_stripes(2, 128*1024, 0, 256*1024, stripes);
  397. assert(stripes[0].req_start == 0 && stripes[0].req_end == 128*1024);
  398. assert(stripes[1].req_start == 0 && stripes[1].req_end == 128*1024);
  399. assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
  400. // Test 10.1
  401. void *write_buf = malloc(256*1024);
  402. void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 3, 2, 3, write_osd_set, 128*1024, 0);
  403. assert(rmw_buf);
  404. assert(stripes[0].read_start == 0 && stripes[0].read_end == 0);
  405. assert(stripes[1].read_start == 0 && stripes[1].read_end == 0);
  406. assert(stripes[2].read_start == 0 && stripes[2].read_end == 0);
  407. assert(stripes[0].write_start == 0 && stripes[0].write_end == 128*1024);
  408. assert(stripes[1].write_start == 0 && stripes[1].write_end == 128*1024);
  409. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  410. assert(stripes[0].read_buf == NULL);
  411. assert(stripes[1].read_buf == NULL);
  412. assert(stripes[2].read_buf == NULL);
  413. assert(stripes[0].write_buf == write_buf);
  414. assert(stripes[1].write_buf == write_buf+128*1024);
  415. assert(stripes[2].write_buf == rmw_buf);
  416. // Test 10.2
  417. set_pattern(stripes[0].write_buf, 128*1024, PATTERN1);
  418. set_pattern(stripes[1].write_buf, 128*1024, PATTERN2);
  419. calc_rmw_parity_xor(stripes, 3, osd_set, write_osd_set, 128*1024, 0);
  420. assert(stripes[0].write_start == 0 && stripes[0].write_end == 128*1024);
  421. assert(stripes[1].write_start == 0 && stripes[1].write_end == 128*1024);
  422. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  423. assert(stripes[0].write_buf == write_buf);
  424. assert(stripes[1].write_buf == write_buf+128*1024);
  425. assert(stripes[2].write_buf == rmw_buf);
  426. check_pattern(stripes[2].write_buf, 128*1024, PATTERN1^PATTERN2);
  427. free(rmw_buf);
  428. free(write_buf);
  429. }
  430. /***
  431. 11. partial recovery case:
  432. calc_rmw(offset=128K, len=128K, read_osd_set=[1,0,0], write_osd_set=[1,2,3])
  433. = {
  434. read: [ [ 0, 128K ], [ 0, 0 ], [ 0, 0 ] ],
  435. write: [ [ 0, 0 ], [ 0, 128K ], [ 0, 128K ] ],
  436. input buffer: [ write1 ],
  437. rmw buffer: [ write2, read0 ],
  438. }
  439. then, after calc_rmw_parity_xor(): all the same
  440. + check write2 buffer
  441. ***/
  442. void test11()
  443. {
  444. osd_num_t osd_set[3] = { 1, 0, 0 };
  445. osd_num_t write_osd_set[3] = { 1, 2, 3 };
  446. osd_rmw_stripe_t stripes[3] = { 0 };
  447. // Test 11.0
  448. split_stripes(2, 128*1024, 128*1024, 256*1024, stripes);
  449. assert(stripes[0].req_start == 0 && stripes[0].req_end == 0);
  450. assert(stripes[1].req_start == 0 && stripes[1].req_end == 128*1024);
  451. assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
  452. // Test 11.1
  453. void *write_buf = malloc(256*1024);
  454. void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 3, 2, 3, write_osd_set, 128*1024, 0);
  455. assert(rmw_buf);
  456. assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024);
  457. assert(stripes[1].read_start == 0 && stripes[1].read_end == 0);
  458. assert(stripes[2].read_start == 0 && stripes[2].read_end == 0);
  459. assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
  460. assert(stripes[1].write_start == 0 && stripes[1].write_end == 128*1024);
  461. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  462. assert(stripes[0].read_buf == rmw_buf+128*1024);
  463. assert(stripes[1].read_buf == NULL);
  464. assert(stripes[2].read_buf == NULL);
  465. assert(stripes[0].write_buf == NULL);
  466. assert(stripes[1].write_buf == write_buf);
  467. assert(stripes[2].write_buf == rmw_buf);
  468. // Test 11.2
  469. set_pattern(stripes[0].read_buf, 128*1024, PATTERN1);
  470. set_pattern(stripes[1].write_buf, 128*1024, PATTERN2);
  471. calc_rmw_parity_xor(stripes, 3, osd_set, write_osd_set, 128*1024, 0);
  472. assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
  473. assert(stripes[1].write_start == 0 && stripes[1].write_end == 128*1024);
  474. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  475. assert(stripes[0].write_buf == NULL);
  476. assert(stripes[1].write_buf == write_buf);
  477. assert(stripes[2].write_buf == rmw_buf);
  478. check_pattern(stripes[2].write_buf, 128*1024, PATTERN1^PATTERN2);
  479. free(rmw_buf);
  480. free(write_buf);
  481. }
  482. /***
  483. 12. parity recovery case:
  484. calc_rmw(offset=0, len=0, read_osd_set=[1,2,0], write_osd_set=[1,2,3])
  485. = {
  486. read: [ [ 0, 128K ], [ 0, 128K ], [ 0, 0 ] ],
  487. write: [ [ 0, 0 ], [ 0, 0 ], [ 0, 128K ] ],
  488. input buffer: [],
  489. rmw buffer: [ write2, read0, read1 ],
  490. }
  491. then, after calc_rmw_parity_xor(): all the same
  492. + check write2 buffer
  493. ***/
  494. void test12()
  495. {
  496. osd_num_t osd_set[3] = { 1, 2, 0 };
  497. osd_num_t write_osd_set[3] = { 1, 2, 3 };
  498. osd_rmw_stripe_t stripes[3] = { 0 };
  499. // Test 12.0
  500. split_stripes(2, 128*1024, 0, 0, stripes);
  501. assert(stripes[0].req_start == 0 && stripes[0].req_end == 0);
  502. assert(stripes[1].req_start == 0 && stripes[1].req_end == 0);
  503. assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
  504. // Test 12.1
  505. void *rmw_buf = calc_rmw(NULL, stripes, osd_set, 3, 2, 3, write_osd_set, 128*1024, 0);
  506. assert(rmw_buf);
  507. assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024);
  508. assert(stripes[1].read_start == 0 && stripes[1].read_end == 128*1024);
  509. assert(stripes[2].read_start == 0 && stripes[2].read_end == 0);
  510. assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
  511. assert(stripes[1].write_start == 0 && stripes[1].write_end == 0);
  512. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  513. assert(stripes[0].read_buf == rmw_buf+128*1024);
  514. assert(stripes[1].read_buf == rmw_buf+2*128*1024);
  515. assert(stripes[2].read_buf == NULL);
  516. assert(stripes[0].write_buf == NULL);
  517. assert(stripes[1].write_buf == NULL);
  518. assert(stripes[2].write_buf == rmw_buf);
  519. // Test 12.2
  520. set_pattern(stripes[0].read_buf, 128*1024, PATTERN1);
  521. set_pattern(stripes[1].read_buf, 128*1024, PATTERN2);
  522. calc_rmw_parity_xor(stripes, 3, osd_set, write_osd_set, 128*1024, 0);
  523. assert(stripes[0].write_start == 0 && stripes[0].write_end == 0);
  524. assert(stripes[1].write_start == 0 && stripes[1].write_end == 0);
  525. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  526. assert(stripes[0].write_buf == NULL);
  527. assert(stripes[1].write_buf == NULL);
  528. assert(stripes[2].write_buf == rmw_buf);
  529. check_pattern(stripes[2].write_buf, 128*1024, PATTERN1^PATTERN2);
  530. free(rmw_buf);
  531. }
  532. /***
  533. 13. basic jerasure 2+2 test
  534. calc_rmw(offset=128K-4K, len=8K, osd_set=[1,2,0,0], write_set=[1,2,3,4])
  535. = {
  536. read: [ [ 0, 128K ], [ 0, 128K ], [ 0, 0 ], [ 0, 0 ] ],
  537. write: [ [ 128K-4K, 128K ], [ 0, 4K ], [ 0, 128K ], [ 0, 128K ] ],
  538. input buffer: [ write0, write1 ],
  539. rmw buffer: [ write2, write3, read0, read1 ],
  540. }
  541. then, after calc_rmw_parity_jerasure(): all the same
  542. then simulate read with read_osd_set=[0,0,3,4] and check read0,read1 buffers
  543. ***/
  544. void test13()
  545. {
  546. use_jerasure(4, 2, true);
  547. osd_num_t osd_set[4] = { 1, 2, 0, 0 };
  548. osd_num_t write_osd_set[4] = { 1, 2, 3, 4 };
  549. osd_rmw_stripe_t stripes[4] = { 0 };
  550. // Test 13.0
  551. void *write_buf = malloc_or_die(8192);
  552. split_stripes(2, 128*1024, 128*1024-4096, 8192, stripes);
  553. assert(stripes[0].req_start == 128*1024-4096 && stripes[0].req_end == 128*1024);
  554. assert(stripes[1].req_start == 0 && stripes[1].req_end == 4096);
  555. assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
  556. assert(stripes[3].req_start == 0 && stripes[3].req_end == 0);
  557. // Test 13.1
  558. void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 4, 2, 4, write_osd_set, 128*1024, 0);
  559. assert(rmw_buf);
  560. assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024-4096);
  561. assert(stripes[1].read_start == 4096 && stripes[1].read_end == 128*1024);
  562. assert(stripes[2].read_start == 0 && stripes[2].read_end == 0);
  563. assert(stripes[3].read_start == 0 && stripes[3].read_end == 0);
  564. assert(stripes[0].write_start == 128*1024-4096 && stripes[0].write_end == 128*1024);
  565. assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);
  566. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  567. assert(stripes[3].write_start == 0 && stripes[3].write_end == 128*1024);
  568. assert(stripes[0].read_buf == rmw_buf+2*128*1024);
  569. assert(stripes[1].read_buf == rmw_buf+3*128*1024-4096);
  570. assert(stripes[2].read_buf == NULL);
  571. assert(stripes[3].read_buf == NULL);
  572. assert(stripes[0].write_buf == write_buf);
  573. assert(stripes[1].write_buf == write_buf+4096);
  574. assert(stripes[2].write_buf == rmw_buf);
  575. assert(stripes[3].write_buf == rmw_buf+128*1024);
  576. // Test 13.2 - encode
  577. set_pattern(write_buf, 8192, PATTERN3);
  578. set_pattern(stripes[0].read_buf, 128*1024-4096, PATTERN1);
  579. set_pattern(stripes[1].read_buf, 128*1024-4096, PATTERN2);
  580. calc_rmw_parity_jerasure(stripes, 4, 2, osd_set, write_osd_set, 128*1024, 0);
  581. assert(stripes[0].write_start == 128*1024-4096 && stripes[0].write_end == 128*1024);
  582. assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);
  583. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  584. assert(stripes[3].write_start == 0 && stripes[3].write_end == 128*1024);
  585. assert(stripes[0].write_buf == write_buf);
  586. assert(stripes[1].write_buf == write_buf+4096);
  587. assert(stripes[2].write_buf == rmw_buf);
  588. assert(stripes[3].write_buf == rmw_buf+128*1024);
  589. // Test 13.3 - full decode and verify
  590. osd_num_t read_osd_set[4] = { 0, 0, 3, 4 };
  591. memset(stripes, 0, sizeof(stripes));
  592. split_stripes(2, 128*1024, 0, 256*1024, stripes);
  593. assert(stripes[0].req_start == 0 && stripes[0].req_end == 128*1024);
  594. assert(stripes[1].req_start == 0 && stripes[1].req_end == 128*1024);
  595. assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
  596. assert(stripes[3].req_start == 0 && stripes[3].req_end == 0);
  597. for (int role = 0; role < 4; role++)
  598. {
  599. stripes[role].read_start = stripes[role].req_start;
  600. stripes[role].read_end = stripes[role].req_end;
  601. }
  602. assert(extend_missing_stripes(stripes, read_osd_set, 2, 4) == 0);
  603. assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024);
  604. assert(stripes[1].read_start == 0 && stripes[1].read_end == 128*1024);
  605. assert(stripes[2].read_start == 0 && stripes[2].read_end == 128*1024);
  606. assert(stripes[3].read_start == 0 && stripes[3].read_end == 128*1024);
  607. void *read_buf = alloc_read_buffer(stripes, 4, 0);
  608. assert(read_buf);
  609. assert(stripes[0].read_buf == read_buf);
  610. assert(stripes[1].read_buf == read_buf+128*1024);
  611. assert(stripes[2].read_buf == read_buf+2*128*1024);
  612. assert(stripes[3].read_buf == read_buf+3*128*1024);
  613. memcpy(read_buf+2*128*1024, rmw_buf, 128*1024);
  614. memcpy(read_buf+3*128*1024, rmw_buf+128*1024, 128*1024);
  615. reconstruct_stripes_jerasure(stripes, 4, 2, 0);
  616. check_pattern(stripes[0].read_buf, 128*1024-4096, PATTERN1);
  617. check_pattern(stripes[0].read_buf+128*1024-4096, 4096, PATTERN3);
  618. check_pattern(stripes[1].read_buf, 4096, PATTERN3);
  619. check_pattern(stripes[1].read_buf+4096, 128*1024-4096, PATTERN2);
  620. free(read_buf);
  621. // Test 13.4 - partial decode (only 1st chunk) and verify
  622. memset(stripes, 0, sizeof(stripes));
  623. split_stripes(2, 128*1024, 0, 128*1024, stripes);
  624. assert(stripes[0].req_start == 0 && stripes[0].req_end == 128*1024);
  625. assert(stripes[1].req_start == 0 && stripes[1].req_end == 0);
  626. assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
  627. assert(stripes[3].req_start == 0 && stripes[3].req_end == 0);
  628. for (int role = 0; role < 4; role++)
  629. {
  630. stripes[role].read_start = stripes[role].req_start;
  631. stripes[role].read_end = stripes[role].req_end;
  632. }
  633. assert(extend_missing_stripes(stripes, read_osd_set, 2, 4) == 0);
  634. assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024);
  635. assert(stripes[1].read_start == 0 && stripes[1].read_end == 0);
  636. assert(stripes[2].read_start == 0 && stripes[2].read_end == 128*1024);
  637. assert(stripes[3].read_start == 0 && stripes[3].read_end == 128*1024);
  638. read_buf = alloc_read_buffer(stripes, 4, 0);
  639. assert(read_buf);
  640. assert(stripes[0].read_buf == read_buf);
  641. assert(stripes[1].read_buf == NULL);
  642. assert(stripes[2].read_buf == read_buf+128*1024);
  643. assert(stripes[3].read_buf == read_buf+2*128*1024);
  644. memcpy(read_buf+128*1024, rmw_buf, 128*1024);
  645. memcpy(read_buf+2*128*1024, rmw_buf+128*1024, 128*1024);
  646. reconstruct_stripes_jerasure(stripes, 4, 2, 0);
  647. check_pattern(stripes[0].read_buf, 128*1024-4096, PATTERN1);
  648. check_pattern(stripes[0].read_buf+128*1024-4096, 4096, PATTERN3);
  649. free(read_buf);
  650. // Huh done
  651. free(rmw_buf);
  652. free(write_buf);
  653. use_jerasure(4, 2, false);
  654. }
  655. /***
  656. 13. basic jerasure 2+1 test
  657. calc_rmw(offset=128K-4K, len=8K, osd_set=[1,2,0], write_set=[1,2,3])
  658. = {
  659. read: [ [ 0, 128K ], [ 0, 128K ], [ 0, 0 ] ],
  660. write: [ [ 128K-4K, 128K ], [ 0, 4K ], [ 0, 128K ] ],
  661. input buffer: [ write0, write1 ],
  662. rmw buffer: [ write2, read0, read1 ],
  663. }
  664. then, after calc_rmw_parity_jerasure(): all the same
  665. then simulate read with read_osd_set=[0,2,3] and check read0 buffer
  666. ***/
  667. void test14()
  668. {
  669. const int bmp = 4;
  670. use_jerasure(3, 2, true);
  671. osd_num_t osd_set[3] = { 1, 2, 0 };
  672. osd_num_t write_osd_set[3] = { 1, 2, 3 };
  673. osd_rmw_stripe_t stripes[3] = { 0 };
  674. unsigned bitmaps[3] = { 0 };
  675. // Test 13.0
  676. void *write_buf = malloc_or_die(8192);
  677. split_stripes(2, 128*1024, 128*1024-4096, 8192, stripes);
  678. assert(stripes[0].req_start == 128*1024-4096 && stripes[0].req_end == 128*1024);
  679. assert(stripes[1].req_start == 0 && stripes[1].req_end == 4096);
  680. assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
  681. // Test 13.1
  682. void *rmw_buf = calc_rmw(write_buf, stripes, osd_set, 3, 2, 3, write_osd_set, 128*1024, bmp);
  683. for (int i = 0; i < 3; i++)
  684. stripes[i].bmp_buf = bitmaps+i;
  685. assert(rmw_buf);
  686. assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024-4096);
  687. assert(stripes[1].read_start == 4096 && stripes[1].read_end == 128*1024);
  688. assert(stripes[2].read_start == 0 && stripes[2].read_end == 0);
  689. assert(stripes[0].write_start == 128*1024-4096 && stripes[0].write_end == 128*1024);
  690. assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);
  691. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  692. assert(stripes[0].read_buf == rmw_buf+128*1024);
  693. assert(stripes[1].read_buf == rmw_buf+2*128*1024-4096);
  694. assert(stripes[2].read_buf == NULL);
  695. assert(stripes[0].write_buf == write_buf);
  696. assert(stripes[1].write_buf == write_buf+4096);
  697. assert(stripes[2].write_buf == rmw_buf);
  698. // Test 13.2 - encode
  699. set_pattern(write_buf, 8192, PATTERN3);
  700. set_pattern(stripes[0].read_buf, 128*1024-4096, PATTERN1);
  701. set_pattern(stripes[1].read_buf, 128*1024-4096, PATTERN2);
  702. memset(stripes[0].bmp_buf, 0, bmp);
  703. memset(stripes[1].bmp_buf, 0, bmp);
  704. memset(stripes[2].bmp_buf, 0, bmp);
  705. calc_rmw_parity_jerasure(stripes, 3, 2, osd_set, write_osd_set, 128*1024, bmp);
  706. assert(*(uint32_t*)stripes[0].bmp_buf == 0x80000000);
  707. assert(*(uint32_t*)stripes[1].bmp_buf == 0x00000001);
  708. assert(*(uint32_t*)stripes[2].bmp_buf == 0x80000001); // jerasure 2+1 is still just XOR
  709. assert(stripes[0].write_start == 128*1024-4096 && stripes[0].write_end == 128*1024);
  710. assert(stripes[1].write_start == 0 && stripes[1].write_end == 4096);
  711. assert(stripes[2].write_start == 0 && stripes[2].write_end == 128*1024);
  712. assert(stripes[0].write_buf == write_buf);
  713. assert(stripes[1].write_buf == write_buf+4096);
  714. assert(stripes[2].write_buf == rmw_buf);
  715. // Test 13.3 - decode and verify
  716. osd_num_t read_osd_set[4] = { 0, 2, 3 };
  717. memset(stripes, 0, sizeof(stripes));
  718. split_stripes(2, 128*1024, 0, 128*1024, stripes);
  719. assert(stripes[0].req_start == 0 && stripes[0].req_end == 128*1024);
  720. assert(stripes[1].req_start == 0 && stripes[1].req_end == 0);
  721. assert(stripes[2].req_start == 0 && stripes[2].req_end == 0);
  722. for (int role = 0; role < 3; role++)
  723. {
  724. stripes[role].read_start = stripes[role].req_start;
  725. stripes[role].read_end = stripes[role].req_end;
  726. }
  727. assert(extend_missing_stripes(stripes, read_osd_set, 2, 3) == 0);
  728. assert(stripes[0].read_start == 0 && stripes[0].read_end == 128*1024);
  729. assert(stripes[1].read_start == 0 && stripes[1].read_end == 128*1024);
  730. assert(stripes[2].read_start == 0 && stripes[2].read_end == 128*1024);
  731. void *read_buf = alloc_read_buffer(stripes, 3, 0);
  732. for (int i = 0; i < 3; i++)
  733. stripes[i].bmp_buf = bitmaps+i;
  734. assert(read_buf);
  735. assert(stripes[0].read_buf == read_buf);
  736. assert(stripes[1].read_buf == read_buf+128*1024);
  737. assert(stripes[2].read_buf == read_buf+2*128*1024);
  738. set_pattern(stripes[1].read_buf, 4096, PATTERN3);
  739. set_pattern(stripes[1].read_buf+4096, 128*1024-4096, PATTERN2);
  740. memcpy(stripes[2].read_buf, rmw_buf, 128*1024);
  741. reconstruct_stripes_jerasure(stripes, 3, 2, bmp);
  742. check_pattern(stripes[0].read_buf, 128*1024-4096, PATTERN1);
  743. check_pattern(stripes[0].read_buf+128*1024-4096, 4096, PATTERN3);
  744. free(read_buf);
  745. // Huh done
  746. free(rmw_buf);
  747. free(write_buf);
  748. use_jerasure(3, 2, false);
  749. }