diff --git a/.dockerignore b/.dockerignore index 61acc8da..06401a9d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,3 +13,5 @@ nbd_proxy rm_inode fio qemu +rpm/*.Dockerfile +debian/*.Dockerfile diff --git a/README.md b/README.md index 54ce9c4e..338a28ea 100644 --- a/README.md +++ b/README.md @@ -31,11 +31,11 @@ breaking changes in the future. However, the following is implemented: - QEMU driver (built out-of-tree) - Loadable fio engine for benchmarks (also built out-of-tree) - NBD proxy for kernel mounts -- Inode removal tool (./rm_inode) +- Inode removal tool (vitastor-rm) +- Packaging for Debian and CentOS ## Roadmap -- Packaging for Debian and, probably, CentOS too - OSD creation tool (OSDs currently have to be created by hand) - Other administrative tools - Per-inode I/O and space usage statistics @@ -280,7 +280,31 @@ Vitastor with single-thread NBD on the same hardware: - Linear write (4M T1Q128): 1266 MB/s (compared to 2800 MB/s via fio) - Linear read (4M T1Q128): 975 MB/s (compared to 1500 MB/s via fio) -## Building +## Installation + +### Debian + +- Trust Vitastor package signing key: + `wget -q -O - https://vitastor.io/debian/pubkey | sudo apt-key add -` +- Add Vitastor package repository to your /etc/apt/sources.list: + - Debian 11 (Bullseye/Sid): `deb https://vitastor.io/debian bullseye main` + - Debian 10 (Buster): `deb https://vitastor.io/debian buster main` +- For Debian 10 (Buster) also enable backports repository: + `deb http://deb.debian.org/debian buster-backports main` +- Install packages: `apt update; apt install vitastor lp-solve etcd linux-image-amd64` + +### CentOS + +- Add Vitastor package repository: + - CentOS 7: `yum install https://vitastor.io/rpms/centos/7/vitastor-release-1.0-1.el7.noarch.rpm` + - CentOS 8: `dnf install https://vitastor.io/rpms/centos/8/vitastor-release-1.0-1.el8.noarch.rpm` +- Enable EPEL: `yum/dnf install epel-release` +- Enable elrepo-kernel: + - CentOS 7: `yum install https://www.elrepo.org/elrepo-release-7.el7.elrepo.noarch.rpm` + - CentOS 8: `dnf install https://www.elrepo.org/elrepo-release-8.el8.elrepo.noarch.rpm` +- Install packages: `yum/dnf install vitastor lpsolve etcd kernel-ml` + +### Building from Source - Install Linux kernel 5.4 or newer, for io_uring support. 5.8 or later is highly recommended because there is at least one known io_uring hang with 5.4 and an HP SmartArray controller. @@ -291,7 +315,7 @@ Vitastor with single-thread NBD on the same hardware: move PGs out of "starting" state if you have at least around ~500 PGs or so. The custom build will be unnecessary when etcd merges the fix: https://github.com/etcd-io/etcd/pull/12402. - Install node.js 10 or newer. -- Install gcc and g++ 8.x or later. +- Install gcc and g++ 8.x or newer. - Clone https://yourcmc.ru/git/vitalif/vitastor/ with submodules. - Install QEMU 2.0+, get its source, begin to build it, stop the build and copy headers: - `/include` → `/qemu/include` @@ -306,9 +330,12 @@ Vitastor with single-thread NBD on the same hardware: * For QEMU 3.0+: `/qapi` → `/qemu/b/qemu/qapi` * For QEMU 2.0+: `/qapi-types.h` → `/qemu/b/qemu/qapi-types.h` - `config-host.h` and `qapi` are required because they contain generated headers +- You can also rebuild QEMU with a patch that makes LD_PRELOAD unnecessary to load vitastor driver. + See `qemu-*.*-vitastor.patch`. - Install fio 3.7 or later, get its source and symlink it into `/fio`. - Build Vitastor with `make -j8`. -- Copy binaries somewhere. +- Run `make install` (optionally with `LIBDIR=/usr/lib64 QEMU_PLUGINDIR=/usr/lib64/qemu-kvm` + if you're using an RPM-based distro). ## Running @@ -323,8 +350,8 @@ and calculate disk offsets almost by hand. This will be fixed in near future. - Create global configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/global '{"immediate_commit":"all"}'` (if all your drives have capacitors). - Create pool configuration in etcd: `etcdctl --endpoints=... put /vitastor/config/pools '{"1":{"name":"testpool","scheme":"replicated","pg_size":2,"pg_minsize":1,"pg_count":256,"failure_domain":"host"}}'`. -- Calculate offsets for your drives with `node ./mon/simple-offsets.js --device /dev/sdX`. -- Make systemd units for your OSDs. Look at `./mon/make-units.sh` for example. +- Calculate offsets for your drives with `node /usr/lib/vitastor/mon/simple-offsets.js --device /dev/sdX`. +- Make systemd units for your OSDs. Look at `/usr/lib/vitastor/mon/make-units.sh` for example. Notable configuration variables from the example: - `disable_data_fsync 1` - only safe with server-grade drives with capacitors. - `immediate_commit all` - use this if all your drives are server-grade. @@ -343,25 +370,25 @@ and calculate disk offsets almost by hand. This will be fixed in near future. setting is set, it is also required to raise `journal_sector_buffer_count` setting, which is the number of dirty journal sectors that may be written to at the same time. - `systemctl start vitastor.target` everywhere. -- Start any number of monitors: `cd mon; node mon-main.js --etcd_url 'http://10.115.0.10:2379,http://10.115.0.11:2379,http://10.115.0.12:2379,http://10.115.0.13:2379' --etcd_prefix '/vitastor' --etcd_start_timeout 5`. +- Start any number of monitors: `node /usr/lib/vitastor/mon/mon-main.js --etcd_url 'http://10.115.0.10:2379,http://10.115.0.11:2379,http://10.115.0.12:2379,http://10.115.0.13:2379' --etcd_prefix '/vitastor' --etcd_start_timeout 5`. - At this point, one of the monitors will configure PGs and OSDs will start them. - You can check PG states with `etcdctl --endpoints=... get --prefix /vitastor/pg/state`. All PGs should become 'active'. -- Run tests with (for example): `fio -thread -ioengine=./libfio_cluster.so -name=test -bs=4M -direct=1 -iodepth=16 -rw=write -etcd=10.115.0.10:2379/v3 -pool=1 -inode=1 -size=400G`. +- Run tests with (for example): `fio -thread -ioengine=/usr/lib/x86_64-linux-gnu/vitastor/libfio_cluster.so -name=test -bs=4M -direct=1 -iodepth=16 -rw=write -etcd=10.115.0.10:2379/v3 -pool=1 -inode=1 -size=400G`. - Upload VM disk image with qemu-img (for example): ``` - LD_PRELOAD=./qemu_driver.so qemu-img convert -f qcow2 debian10.qcow2 -p + LD_PRELOAD=/usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so qemu-img convert -f qcow2 debian10.qcow2 -p -O raw 'vitastor:etcd_host=10.115.0.10\:2379/v3:pool=1:inode=1:size=2147483648' ``` - Run QEMU with (for example): ``` - LD_PRELOAD=./qemu_driver.so qemu-system-x86_64 -enable-kvm -m 1024 + LD_PRELOAD=/usr/lib/x86_64-linux-gnu/qemu/block-vitastor.so qemu-system-x86_64 -enable-kvm -m 1024 -drive 'file=vitastor:etcd_host=10.115.0.10\:2379/v3:pool=1:inode=1:size=2147483648',format=raw,if=none,id=drive-virtio-disk0,cache=none -device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x5,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1,write-cache=off,physical_block_size=4096,logical_block_size=512 -vnc 0.0.0.0:0 ``` - Remove inode with (for example): ``` - ./rm_inode --etcd_address 10.115.0.10:2379/v3 --pool 1 --inode 1 --parallel_osds 16 --iodepth 32 + vitastor-rm --etcd_address 10.115.0.10:2379/v3 --pool 1 --inode 1 --parallel_osds 16 --iodepth 32 ``` ## Known Problems diff --git a/debian/vitastor-bullseye.Dockerfile b/debian/vitastor-bullseye.Dockerfile index e28ef8a0..fab99891 100644 --- a/debian/vitastor-bullseye.Dockerfile +++ b/debian/vitastor-bullseye.Dockerfile @@ -38,13 +38,13 @@ RUN set -e; \ V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)(~bpo[\d\+]*)?\).*$/$1/')+vitastor1; \ echo ">>> VERSION: $V"; \ DEBFULLNAME="Vitaliy Filippov " dch -D $REL -v $V 'Plug Vitastor block driver'; \ - DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto; \ + DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \ rm -rf /root/build/qemu-$REL/qemu-*/ RUN cd /root/build/qemu-$REL && apt-get -y install ./qemu-system-data*.deb ./qemu-system-common_*.deb ./qemu-system-x86_*.deb ./qemu_*.deb ADD . /root/vitastor -RUN set -e; \ +RUN set -e -x; \ mkdir -p /root/fio-build/; \ cd /root/fio-build/; \ rm -rf /root/fio-build/*; \ @@ -63,15 +63,24 @@ RUN set -e; \ ln -s /root/build/qemu-$REL/qemu-*/ vitastor-0.5/qemu; \ ln -s /root/fio-build/fio-*/ vitastor-0.5/fio; \ cd vitastor-0.5; \ + FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \ + QEMU=$(head -n1 qemu/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \ sh copy-qemu-includes.sh; \ sh copy-fio-includes.sh; \ rm qemu fio; \ - mv qemu-copy qemu; \ - mv fio-copy fio; \ + mkdir -p a b debian/patches; \ + mv qemu-copy b/qemu; \ + mv fio-copy b/fio; \ + diff -NaurpbB a b > debian/patches/qemu-fio-headers.patch || true; \ + echo qemu-fio-headers.patch >> debian/patches/series; \ + rm -rf a b; \ rm -rf /root/build/qemu-$REL/qemu*/; \ + echo "dep:fio=$FIO" > debian/substvars; \ + echo "dep:qemu=$QEMU" >> debian/substvars; \ cd /root/build/vitastor-$REL; \ - tar -czf vitastor_0.5.orig.tar.gz vitastor-0.5; \ + tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.5.orig.tar.xz vitastor-0.5; \ cd vitastor-0.5; \ - ls -l; \ - DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto; \ + V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \ + DEBFULLNAME="Vitaliy Filippov " dch -D $REL -v "$V""$REL" "Rebuild for $REL"; \ + DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \ rm -rf /root/build/vitastor-$REL/vitastor-*/ diff --git a/debian/vitastor-buster.Dockerfile b/debian/vitastor-buster.Dockerfile index d69e9364..ab2b2bc8 100644 --- a/debian/vitastor-buster.Dockerfile +++ b/debian/vitastor-buster.Dockerfile @@ -32,13 +32,13 @@ RUN set -e; \ cd /root/build/qemu-buster/qemu-*/; \ V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)(~bpo[\d\+]*)\).*$/$1/')+vitastor1; \ DEBFULLNAME="Vitaliy Filippov " dch -D buster -v $V 'Plug Vitastor block driver'; \ - DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto; \ + DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \ rm -rf /root/build/qemu-buster/qemu-*/ RUN cd /root/build/qemu-buster && apt-get -y -t buster-backports install ./qemu-system-data*.deb ./qemu-system-common_*.deb ./qemu-system-x86_*.deb ./qemu_*.deb ADD . /root/vitastor -RUN set -e; \ +RUN set -e -x; \ mkdir -p /root/fio-build/; \ cd /root/fio-build/; \ rm -rf /root/fio-build/*; \ @@ -57,15 +57,24 @@ RUN set -e; \ ln -s /root/build/qemu-buster/qemu-*/ vitastor-0.5/qemu; \ ln -s /root/fio-build/fio-*/ vitastor-0.5/fio; \ cd vitastor-0.5; \ + FIO=$(head -n1 fio/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \ + QEMU=$(head -n1 qemu/debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \ sh copy-qemu-includes.sh; \ sh copy-fio-includes.sh; \ rm qemu fio; \ - mv qemu-copy qemu; \ - mv fio-copy fio; \ + mkdir -p a b debian/patches; \ + mv qemu-copy b/qemu; \ + mv fio-copy b/fio; \ + diff -NaurpbB a b > debian/patches/qemu-fio-headers.patch || true; \ + echo qemu-fio-headers.patch >> debian/patches/series; \ + rm -rf a b; \ rm -rf /root/build/qemu-buster/qemu*/; \ + echo "dep:fio=$FIO" > debian/substvars; \ + echo "dep:qemu=$QEMU" >> debian/substvars; \ cd /root/build/vitastor-buster; \ - tar -czf vitastor_0.5.orig.tar.gz vitastor-0.5; \ + tar --sort=name --mtime='2020-01-01' --owner=0 --group=0 --exclude=debian -cJf vitastor_0.5.orig.tar.xz vitastor-0.5; \ cd vitastor-0.5; \ - ls -l; \ - DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto; \ + V=$(head -n1 debian/changelog | perl -pe 's/^.*\((.*?)\).*$/$1/'); \ + DEBFULLNAME="Vitaliy Filippov " dch -D buster -v "$V""buster" "Rebuild for buster"; \ + DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage --jobs=auto -sa; \ rm -rf /root/build/vitastor-buster/vitastor-*/ diff --git a/rpm/build-tarball.sh b/rpm/build-tarball.sh index aa50861f..33c4974c 100755 --- a/rpm/build-tarball.sh +++ b/rpm/build-tarball.sh @@ -44,8 +44,8 @@ ln -s ~/rpmbuild/BUILD/fio*/ fio sh copy-fio-includes.sh rm fio mv fio-copy fio -FIO=`rpm -qa fio | perl -pe 's/^fio-(.*)$/$1/'` -QEMU=`rpm -qa qemu qemu-kvm | perl -pe 's/^qemu-(?:kvm-)?(.*)$/$1/'` +FIO=`rpm -qi fio | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'` +QEMU=`rpm -qi qemu qemu-kvm | perl -e 'while(<>) { /^Epoch[\s:]+(\S+)/ && print "$1:"; /^Version[\s:]+(\S+)/ && print $1; /^Release[\s:]+(\S+)/ && print "-$1"; }'` perl -i -pe 's/(Requires:\s*fio)([^\n]+)?/$1 = '$FIO'/' $VITASTOR/rpm/vitastor-el$EL.spec -perl -i -pe 's/(Requires:\s*qemu)([^\n]+)?/$1 = '$QEMU'/' $VITASTOR/rpm/vitastor-el$EL.spec +perl -i -pe 's/(Requires:\s*qemu(?:-kvm)?)([^\n]+)?/$1 = '$QEMU'/' $VITASTOR/rpm/vitastor-el$EL.spec tar --transform 's#^#vitastor-0.5/#' --exclude 'rpm/*.rpm' -czf $VITASTOR/../vitastor-0.5$(rpm --eval '%dist').tar.gz * diff --git a/rpm/prepare-env.sh b/rpm/prepare-env.sh deleted file mode 100755 index 843d1fd1..00000000 --- a/rpm/prepare-env.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Install/get required packages on CentOS 7 and 8 - -set -e - -DIST=`rpm --eval '%dist'` -rm -f /etc/yum.repos.d/CentOS-Media.repo -if [ "$DIST" = ".el8" ]; then - dnf -y install centos-release-advanced-virtualization epel-release - dnf --enablerepo='*' -y install gcc-toolset-9 gcc-toolset-9-gcc-c++ gperftools-devel qemu-kvm fio nodejs rpm-build - dnf download --disablerepo='*' --enablerepo='centos-advanced-virtualization-source' --source qemu-kvm - dnf download --source fio - rpm --nomd5 -i qemu*.src.rpm - rpm --nomd5 -i fio*.src.rpm - cd ~/rpmbuild/SPECS - dnf builddep -y --enablerepo='*' --spec qemu-kvm.spec - dnf builddep -y --enablerepo='*' --spec fio.spec -else - yum -y --enablerepo=extras install centos-release-scl epel-release yum-utils rpm-build - yum -y install devtoolset-9-gcc-c++ devtoolset-9-libatomic-devel gperftools-devel qemu fio rh-nodejs12 - yumdownloader --source qemu - yumdownloader --source fio - rpm --nomd5 -i qemu*.src.rpm - rpm --nomd5 -i fio*.src.rpm - cd ~/rpmbuild/SPECS - yum-builddep -y --enablerepo='*' qemu.spec - yum-builddep -y --enablerepo='*' fio.spec -fi diff --git a/rpm/vitastor-el7.spec b/rpm/vitastor-el7.spec index b27b398c..7e8b3032 100644 --- a/rpm/vitastor-el7.spec +++ b/rpm/vitastor-el7.spec @@ -41,7 +41,8 @@ rm -rf $RPM_BUILD_ROOT cd mon npm install cd .. -cp -r mon %buildroot%_libdir/vitastor/mon +mkdir -p %buildroot/usr/lib/vitastor +cp -r mon %buildroot/usr/lib/vitastor/mon %files @@ -52,6 +53,7 @@ cp -r mon %buildroot%_libdir/vitastor/mon %_bindir/vitastor-rm %_libdir/qemu-kvm/block-vitastor.so %_libdir/vitastor +/usr/lib/vitastor %changelog diff --git a/rpm/vitastor-el8.Dockerfile b/rpm/vitastor-el8.Dockerfile index fd835c00..95440dc6 100644 --- a/rpm/vitastor-el8.Dockerfile +++ b/rpm/vitastor-el8.Dockerfile @@ -29,7 +29,7 @@ RUN set -e; \ RUN rpm -i `ls /root/build/liburing-el7/liburing-*.x86_64.rpm | grep -v debug` -ADD . /root/vitastor +ADD qemu-*-vitastor.patch /root/vitastor/ RUN set -e; \ mkdir -p /root/build/qemu-el8; \ @@ -48,6 +48,10 @@ RUN set -e; \ cp ~/rpmbuild/RPMS/*/*qemu* /root/build/qemu-el8/; \ cp ~/rpmbuild/SRPMS/*qemu* /root/build/qemu-el8/ +RUN cd /root/build/qemu-el8; dnf -y install `ls qemu*.rpm | grep -vP 'debug|guest|tests|src'` + +ADD . /root/vitastor + RUN set -e; \ cd /root/vitastor/rpm; \ sh build-tarball.sh; \ diff --git a/rpm/vitastor-el8.spec b/rpm/vitastor-el8.spec index d5ed9567..20317089 100644 --- a/rpm/vitastor-el8.spec +++ b/rpm/vitastor-el8.spec @@ -12,7 +12,7 @@ BuildRequires: gperftools-devel BuildRequires: gcc-toolset-9-gcc-c++ BuildRequires: nodejs >= 10 Requires: fio = 3.7-3.el8 -Requires: qemu = 4.2.0-29.el8.6 +Requires: qemu-kvm = 4.2.0-29.el8.6 Requires: nodejs >= 10 Requires: liburing >= 0.6 @@ -38,7 +38,8 @@ rm -rf $RPM_BUILD_ROOT cd mon npm install cd .. -cp -r mon %buildroot%_libdir/vitastor/mon +mkdir -p %buildroot/usr/lib/vitastor +cp -r mon %buildroot/usr/lib/vitastor %files @@ -49,6 +50,7 @@ cp -r mon %buildroot%_libdir/vitastor/mon %_bindir/vitastor-rm %_libdir/qemu-kvm/block-vitastor.so %_libdir/vitastor +/usr/lib/vitastor %changelog