Simplified distributed block storage with strong consistency, like in Ceph
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

196 lines
14 KiB

  1. BINDIR ?= /usr/bin
  2. LIBDIR ?= /usr/lib/x86_64-linux-gnu
  3. QEMU_PLUGINDIR ?= /usr/lib/x86_64-linux-gnu/qemu
  4. BLOCKSTORE_OBJS := allocator.o blockstore.o blockstore_impl.o blockstore_init.o blockstore_open.o blockstore_journal.o blockstore_read.o \
  5. blockstore_write.o blockstore_sync.o blockstore_stable.o blockstore_rollback.o blockstore_flush.o crc32c.o ringloop.o
  6. # -fsanitize=address
  7. CXXFLAGS := -g -O3 -Wall -Wno-sign-compare -Wno-comment -Wno-parentheses -Wno-pointer-arith -fPIC -fdiagnostics-color=always -I/usr/include/jerasure
  8. all: libfio_blockstore.so osd libfio_sec_osd.so libfio_cluster.so stub_osd stub_uring_osd stub_bench osd_test dump_journal qemu_driver.so nbd_proxy rm_inode
  9. clean:
  10. rm -f *.o libblockstore.so libfio_blockstore.so osd libfio_sec_osd.so libfio_cluster.so stub_osd stub_uring_osd stub_bench osd_test dump_journal qemu_driver.so nbd_proxy rm_inode
  11. install: all
  12. mkdir -p $(DESTDIR)$(LIBDIR)/vitastor
  13. install -m 0755 libfio_sec_osd.so $(DESTDIR)$(LIBDIR)/vitastor/
  14. install -m 0755 libfio_cluster.so $(DESTDIR)$(LIBDIR)/vitastor/
  15. install -m 0755 libfio_blockstore.so $(DESTDIR)$(LIBDIR)/vitastor/
  16. install -m 0755 libblockstore.so $(DESTDIR)$(LIBDIR)/vitastor/
  17. mkdir -p $(DESTDIR)$(BINDIR)
  18. install -m 0755 osd $(DESTDIR)$(BINDIR)/vitastor-osd
  19. install -m 0755 dump_journal $(DESTDIR)$(BINDIR)/vitastor-dump-journal
  20. install -m 0755 nbd_proxy $(DESTDIR)$(BINDIR)/vitastor-nbd
  21. install -m 0755 rm_inode $(DESTDIR)$(BINDIR)/vitastor-rm
  22. mkdir -p $(DESTDIR)$(QEMU_PLUGINDIR)
  23. install -m 0755 qemu_driver.so $(DESTDIR)$(QEMU_PLUGINDIR)/block-vitastor.so
  24. dump_journal: dump_journal.cpp crc32c.o blockstore_journal.h
  25. g++ $(CXXFLAGS) -o $@ $< crc32c.o
  26. libblockstore.so: $(BLOCKSTORE_OBJS)
  27. g++ $(CXXFLAGS) -o $@ -shared $(BLOCKSTORE_OBJS) -ltcmalloc_minimal -luring
  28. libfio_blockstore.so: ./libblockstore.so fio_engine.o json11.o
  29. g++ $(CXXFLAGS) -Wl,-rpath,'$(LIBDIR)/vitastor',-rpath,'$$ORIGIN' -shared -o $@ fio_engine.o json11.o libblockstore.so -ltcmalloc_minimal -luring
  30. OSD_OBJS := osd.o osd_secondary.o msgr_receive.o msgr_send.o osd_peering.o osd_flush.o osd_peering_pg.o \
  31. osd_primary.o osd_primary_subops.o etcd_state_client.o messenger.o osd_cluster.o http_client.o osd_ops.o pg_states.o \
  32. osd_rmw.o json11.o base64.o timerfd_manager.o epoll_manager.o
  33. osd: ./libblockstore.so osd_main.cpp osd.h osd_ops.h $(OSD_OBJS)
  34. g++ $(CXXFLAGS) -Wl,-rpath,'$(LIBDIR)/vitastor',-rpath,'$$ORIGIN' -o $@ osd_main.cpp $(OSD_OBJS) libblockstore.so -ltcmalloc_minimal -luring -lJerasure
  35. stub_osd: stub_osd.o rw_blocking.o
  36. g++ $(CXXFLAGS) -o $@ stub_osd.o rw_blocking.o -ltcmalloc_minimal
  37. osd_rmw_test: osd_rmw_test.o allocator.o
  38. g++ $(CXXFLAGS) -o $@ osd_rmw_test.o allocator.o -lJerasure -fsanitize=address
  39. STUB_URING_OSD_OBJS := stub_uring_osd.o epoll_manager.o messenger.o msgr_send.o msgr_receive.o ringloop.o timerfd_manager.o json11.o
  40. stub_uring_osd: $(STUB_URING_OSD_OBJS)
  41. g++ $(CXXFLAGS) -o $@ -ltcmalloc_minimal $(STUB_URING_OSD_OBJS) -luring
  42. stub_bench: stub_bench.cpp osd_ops.h rw_blocking.o
  43. g++ $(CXXFLAGS) -o $@ stub_bench.cpp rw_blocking.o -ltcmalloc_minimal
  44. osd_test: osd_test.cpp osd_ops.h rw_blocking.o
  45. g++ $(CXXFLAGS) -o $@ osd_test.cpp rw_blocking.o -ltcmalloc_minimal
  46. osd_peering_pg_test: osd_peering_pg_test.cpp osd_peering_pg.o
  47. g++ $(CXXFLAGS) -o $@ $< osd_peering_pg.o -ltcmalloc_minimal
  48. libfio_sec_osd.so: fio_sec_osd.o rw_blocking.o
  49. g++ $(CXXFLAGS) -ltcmalloc_minimal -shared -o $@ fio_sec_osd.o rw_blocking.o
  50. FIO_CLUSTER_OBJS := cluster_client.o epoll_manager.o etcd_state_client.o \
  51. messenger.o msgr_send.o msgr_receive.o ringloop.o json11.o http_client.o osd_ops.o pg_states.o timerfd_manager.o base64.o
  52. libfio_cluster.so: fio_cluster.o $(FIO_CLUSTER_OBJS)
  53. g++ $(CXXFLAGS) -ltcmalloc_minimal -shared -o $@ $< $(FIO_CLUSTER_OBJS) -luring
  54. nbd_proxy: nbd_proxy.o $(FIO_CLUSTER_OBJS)
  55. g++ $(CXXFLAGS) -ltcmalloc_minimal -o $@ $< $(FIO_CLUSTER_OBJS) -luring
  56. rm_inode: rm_inode.o $(FIO_CLUSTER_OBJS)
  57. g++ $(CXXFLAGS) -ltcmalloc_minimal -o $@ $< $(FIO_CLUSTER_OBJS) -luring
  58. qemu_driver.o: qemu_driver.c qemu_proxy.h
  59. gcc -I qemu/b/qemu `pkg-config glib-2.0 --cflags` \
  60. -I qemu/include $(CXXFLAGS) -c -o $@ $<
  61. qemu_driver.so: qemu_driver.o qemu_proxy.o $(FIO_CLUSTER_OBJS)
  62. g++ $(CXXFLAGS) -ltcmalloc_minimal -shared -o $@ $(FIO_CLUSTER_OBJS) qemu_driver.o qemu_proxy.o -luring
  63. test_blockstore: ./libblockstore.so test_blockstore.cpp timerfd_interval.o
  64. g++ $(CXXFLAGS) -Wl,-rpath,'$(LIBDIR)/vitastor',-rpath,'$$ORIGIN' -o test_blockstore test_blockstore.cpp timerfd_interval.o libblockstore.so -ltcmalloc_minimal -luring
  65. test_shit: test_shit.cpp osd_peering_pg.o
  66. g++ $(CXXFLAGS) -o test_shit test_shit.cpp -luring -lm
  67. test_allocator: test_allocator.cpp allocator.o
  68. g++ $(CXXFLAGS) -o test_allocator test_allocator.cpp allocator.o
  69. crc32c.o: crc32c.c crc32c.h
  70. g++ $(CXXFLAGS) -c -o $@ $<
  71. json11.o: json11/json11.cpp
  72. g++ $(CXXFLAGS) -c -o json11.o json11/json11.cpp
  73. # Autogenerated
  74. allocator.o: allocator.cpp allocator.h
  75. g++ $(CXXFLAGS) -c -o $@ $<
  76. base64.o: base64.cpp base64.h
  77. g++ $(CXXFLAGS) -c -o $@ $<
  78. blockstore.o: blockstore.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
  79. g++ $(CXXFLAGS) -c -o $@ $<
  80. blockstore_flush.o: blockstore_flush.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
  81. g++ $(CXXFLAGS) -c -o $@ $<
  82. blockstore_impl.o: blockstore_impl.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
  83. g++ $(CXXFLAGS) -c -o $@ $<
  84. blockstore_init.o: blockstore_init.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
  85. g++ $(CXXFLAGS) -c -o $@ $<
  86. blockstore_journal.o: blockstore_journal.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
  87. g++ $(CXXFLAGS) -c -o $@ $<
  88. blockstore_open.o: blockstore_open.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
  89. g++ $(CXXFLAGS) -c -o $@ $<
  90. blockstore_read.o: blockstore_read.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
  91. g++ $(CXXFLAGS) -c -o $@ $<
  92. blockstore_rollback.o: blockstore_rollback.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
  93. g++ $(CXXFLAGS) -c -o $@ $<
  94. blockstore_stable.o: blockstore_stable.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
  95. g++ $(CXXFLAGS) -c -o $@ $<
  96. blockstore_sync.o: blockstore_sync.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
  97. g++ $(CXXFLAGS) -c -o $@ $<
  98. blockstore_write.o: blockstore_write.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
  99. g++ $(CXXFLAGS) -c -o $@ $<
  100. cluster_client.o: cluster_client.cpp cluster_client.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
  101. g++ $(CXXFLAGS) -c -o $@ $<
  102. dump_journal.o: dump_journal.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h ringloop.h
  103. g++ $(CXXFLAGS) -c -o $@ $<
  104. epoll_manager.o: epoll_manager.cpp epoll_manager.h ringloop.h timerfd_manager.h
  105. g++ $(CXXFLAGS) -c -o $@ $<
  106. etcd_state_client.o: etcd_state_client.cpp base64.h etcd_state_client.h http_client.h json11/json11.hpp object_id.h osd_id.h osd_ops.h pg_states.h timerfd_manager.h
  107. g++ $(CXXFLAGS) -c -o $@ $<
  108. fio_cluster.o: fio_cluster.cpp cluster_client.h epoll_manager.h etcd_state_client.h fio/arch/arch.h fio/fio.h fio/optgroup.h fio_headers.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
  109. g++ $(CXXFLAGS) -c -o $@ $<
  110. fio_engine.o: fio_engine.cpp blockstore.h fio/arch/arch.h fio/fio.h fio/optgroup.h fio_headers.h json11/json11.hpp object_id.h ringloop.h
  111. g++ $(CXXFLAGS) -c -o $@ $<
  112. fio_sec_osd.o: fio_sec_osd.cpp fio/arch/arch.h fio/fio.h fio/optgroup.h fio_headers.h object_id.h osd_id.h osd_ops.h rw_blocking.h
  113. g++ $(CXXFLAGS) -c -o $@ $<
  114. http_client.o: http_client.cpp http_client.h json11/json11.hpp timerfd_manager.h
  115. g++ $(CXXFLAGS) -c -o $@ $<
  116. messenger.o: messenger.cpp json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
  117. g++ $(CXXFLAGS) -c -o $@ $<
  118. msgr_receive.o: msgr_receive.cpp json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
  119. g++ $(CXXFLAGS) -c -o $@ $<
  120. msgr_send.o: msgr_send.cpp json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
  121. g++ $(CXXFLAGS) -c -o $@ $<
  122. nbd_proxy.o: nbd_proxy.cpp cluster_client.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
  123. g++ $(CXXFLAGS) -c -o $@ $<
  124. osd.o: osd.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
  125. g++ $(CXXFLAGS) -c -o $@ $<
  126. osd_cluster.o: osd_cluster.cpp base64.h blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h osd_rmw.h pg_states.h ringloop.h timerfd_manager.h
  127. g++ $(CXXFLAGS) -c -o $@ $<
  128. osd_flush.o: osd_flush.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
  129. g++ $(CXXFLAGS) -c -o $@ $<
  130. osd_main.o: osd_main.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
  131. g++ $(CXXFLAGS) -c -o $@ $<
  132. osd_ops.o: osd_ops.cpp object_id.h osd_id.h osd_ops.h
  133. g++ $(CXXFLAGS) -c -o $@ $<
  134. osd_peering.o: osd_peering.cpp base64.h blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
  135. g++ $(CXXFLAGS) -c -o $@ $<
  136. osd_peering_pg.o: osd_peering_pg.cpp cpp-btree/btree_map.h object_id.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h
  137. g++ $(CXXFLAGS) -c -o $@ $<
  138. osd_peering_pg_test.o: osd_peering_pg_test.cpp cpp-btree/btree_map.h object_id.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h
  139. g++ $(CXXFLAGS) -c -o $@ $<
  140. osd_primary.o: osd_primary.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h osd_primary.h osd_rmw.h pg_states.h ringloop.h timerfd_manager.h
  141. g++ $(CXXFLAGS) -c -o $@ $<
  142. osd_primary_subops.o: osd_primary_subops.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h osd_primary.h osd_rmw.h pg_states.h ringloop.h timerfd_manager.h
  143. g++ $(CXXFLAGS) -c -o $@ $<
  144. osd_rmw.o: osd_rmw.cpp allocator.h malloc_or_die.h object_id.h osd_id.h osd_rmw.h xor.h
  145. g++ $(CXXFLAGS) -c -o $@ $<
  146. osd_rmw_test.o: osd_rmw_test.cpp allocator.h malloc_or_die.h object_id.h osd_id.h osd_rmw.cpp osd_rmw.h test_pattern.h xor.h
  147. g++ $(CXXFLAGS) -c -o $@ $<
  148. osd_secondary.o: osd_secondary.cpp blockstore.h cpp-btree/btree_map.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd.h osd_id.h osd_ops.h osd_peering_pg.h pg_states.h ringloop.h timerfd_manager.h
  149. g++ $(CXXFLAGS) -c -o $@ $<
  150. osd_test.o: osd_test.cpp object_id.h osd_id.h osd_ops.h rw_blocking.h test_pattern.h
  151. g++ $(CXXFLAGS) -c -o $@ $<
  152. pg_states.o: pg_states.cpp pg_states.h
  153. g++ $(CXXFLAGS) -c -o $@ $<
  154. qemu_proxy.o: qemu_proxy.cpp cluster_client.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h qemu_proxy.h ringloop.h timerfd_manager.h
  155. g++ $(CXXFLAGS) -c -o $@ $<
  156. ringloop.o: ringloop.cpp ringloop.h
  157. g++ $(CXXFLAGS) -c -o $@ $<
  158. rm_inode.o: rm_inode.cpp cluster_client.h epoll_manager.h etcd_state_client.h http_client.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h pg_states.h ringloop.h timerfd_manager.h
  159. g++ $(CXXFLAGS) -c -o $@ $<
  160. rw_blocking.o: rw_blocking.cpp rw_blocking.h
  161. g++ $(CXXFLAGS) -c -o $@ $<
  162. stub_bench.o: stub_bench.cpp object_id.h osd_id.h osd_ops.h rw_blocking.h
  163. g++ $(CXXFLAGS) -c -o $@ $<
  164. stub_osd.o: stub_osd.cpp object_id.h osd_id.h osd_ops.h rw_blocking.h
  165. g++ $(CXXFLAGS) -c -o $@ $<
  166. stub_uring_osd.o: stub_uring_osd.cpp epoll_manager.h json11/json11.hpp malloc_or_die.h messenger.h object_id.h osd_id.h osd_ops.h ringloop.h timerfd_manager.h
  167. g++ $(CXXFLAGS) -c -o $@ $<
  168. test_allocator.o: test_allocator.cpp allocator.h
  169. g++ $(CXXFLAGS) -c -o $@ $<
  170. test_blockstore.o: test_blockstore.cpp blockstore.h object_id.h ringloop.h timerfd_interval.h
  171. g++ $(CXXFLAGS) -c -o $@ $<
  172. test_shit.o: test_shit.cpp allocator.h blockstore.h blockstore_flush.h blockstore_impl.h blockstore_init.h blockstore_journal.h cpp-btree/btree_map.h crc32c.h malloc_or_die.h object_id.h osd_id.h osd_ops.h osd_peering_pg.cpp osd_peering_pg.h pg_states.h ringloop.h
  173. g++ $(CXXFLAGS) -c -o $@ $<
  174. timerfd_interval.o: timerfd_interval.cpp ringloop.h timerfd_interval.h
  175. g++ $(CXXFLAGS) -c -o $@ $<
  176. timerfd_manager.o: timerfd_manager.cpp timerfd_manager.h
  177. g++ $(CXXFLAGS) -c -o $@ $<