diff --git a/README-ru.md b/README-ru.md index faa19f398..dc1da7711 100644 --- a/README-ru.md +++ b/README-ru.md @@ -51,13 +51,14 @@ Vitastor на данный момент находится в статусе п - Базовая поддержка OpenStack: драйвер Cinder, патчи для Nova и libvirt - Слияние снапшотов (vitastor-cli {snap-rm,flatten,merge}) - Консольный интерфейс для управления образами (vitastor-cli {ls,create,modify}) +- Плагин для Proxmox ## Планы развития - Поддержка удаления снапшотов (слияния слоёв) - Более корректные скрипты разметки дисков и автоматического запуска OSD - Другие инструменты администрирования -- Плагины для OpenNebula, Proxmox и других облачных систем +- Плагины для OpenNebula и других облачных систем - iSCSI-прокси - Более быстрое переключение при отказах - Фоновая проверка целостности без контрольных сумм (сверка реплик) @@ -537,6 +538,71 @@ for i in ./???-*.yaml; do kubectl apply -f $i; done После этого вы сможете создавать PersistentVolume. Пример смотрите в файле [csi/deploy/example-pvc.yaml](csi/deploy/example-pvc.yaml). +### OpenStack + +Чтобы подключить Vitastor к OpenStack: + +- Установите пакеты vitastor-client, libvirt и QEMU из DEB или RPM репозитория Vitastor +- Примените патч `patches/nova-21.diff` или `patches/nova-23.diff` к вашей инсталляции Nova. + nova-21.diff подходит для Nova 21-22, nova-23.diff подходит для Nova 23-24. +- Скопируйте `patches/cinder-vitastor.py` в инсталляцию Cinder как `cinder/volume/drivers/vitastor.py` +- Создайте тип томов в cinder.conf (см. ниже) +- Перезапустите Cinder и Nova + +Пример конфигурации Cinder: + +``` +[DEFAULT] +enabled_backends = lvmdriver-1, vitastor-testcluster +# ... + +[vitastor-testcluster] +volume_driver = cinder.volume.drivers.vitastor.VitastorDriver +volume_backend_name = vitastor-testcluster +image_volume_cache_enabled = True +volume_clear = none +vitastor_etcd_address = 192.168.7.2:2379 +vitastor_etcd_prefix = +vitastor_config_path = /etc/vitastor/vitastor.conf +vitastor_pool_id = 1 +image_upload_use_cinder_backend = True +``` + +Чтобы помещать в Vitastor Glance-образы, нужно использовать +[https://docs.openstack.org/cinder/pike/admin/blockstorage-volume-backed-image.html](образы на основе томов Cinder), +однако, поддержка этой функции ещё не проверялась. + +### Proxmox + +Чтобы подключить Vitastor к Proxmox Virtual Environment (поддерживаются версии 6.4 и 7.1): + +- Добавьте соответствующий Debian-репозиторий Vitastor в sources.list на хостах Proxmox + (buster для 6.4, bullseye для 7.1) +- Установите пакеты vitastor-client и pve-qemu-kvm из репозитория Vitastor +- Скопируйте файл [patches/PVE_VitastorPlugin.pm](patches/PVE_VitastorPlugin.pm) на хосты + Proxmox как `/usr/share/perl5/PVE/Storage/Custom/VitastorPlugin.pm` +- Определите тип хранилища в `/etc/pve/storage.cfg` (см. ниже) +- Перезапустите демон Proxmox: `systemctl restart pvedaemon` + +Пример `/etc/pve/storage.cfg` (единственная обязательная опция - vitastor_pool, все остальные +перечислены внизу для понимания значений по умолчанию): + +``` +vitastor: vitastor + # Пул, в который будут помещаться образы дисков + vitastor_pool testpool + # Путь к файлу конфигурации + vitastor_config_path /etc/vitastor/vitastor.conf + # Адрес(а) etcd, нужны, только если не указаны в vitastor.conf + vitastor_etcd_address 192.168.7.2:2379/v3 + # Префикс ключей метаданных в etcd + vitastor_etcd_prefix /vitastor + # Префикс имён образов + vitastor_prefix pve/ + # Монтировать образы через NBD прокси, через ядро (нужно только для контейнеров) + vitastor_nbd 0 +``` + ## Известные проблемы - Запросы удаления объектов могут в данный момент приводить к "неполным" объектам в EC-пулах, diff --git a/README.md b/README.md index aa7f6d534..1d9b7928c 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ breaking changes in the future. However, the following is implemented: - Basic OpenStack support: Cinder driver, Nova and libvirt patches - Snapshot merge tool (vitastor-cli {snap-rm,flatten,merge}) - Image management CLI (vitastor-cli {ls,create,modify}) +- Proxmox storage plugin ## Roadmap @@ -486,6 +487,70 @@ for i in ./???-*.yaml; do kubectl apply -f $i; done After that you'll be able to create PersistentVolumes. See example in [csi/deploy/example-pvc.yaml](csi/deploy/example-pvc.yaml). +### OpenStack + +To enable Vitastor support in an OpenStack installation: + +- Install vitastor-client, patched QEMU and libvirt packages from Vitastor DEB or RPM repository +- Use `patches/nova-21.diff` or `patches/nova-23.diff` to patch your Nova installation. + Patch 21 fits Nova 21-22, patch 23 fits Nova 23-24. +- Install `patches/cinder-vitastor.py` as `..../cinder/volume/drivers/vitastor.py` +- Define a volume type in cinder.conf (see below) +- Restart Cinder and Nova + +Cinder volume type configuration example: + +``` +[DEFAULT] +enabled_backends = lvmdriver-1, vitastor-testcluster +# ... + +[vitastor-testcluster] +volume_driver = cinder.volume.drivers.vitastor.VitastorDriver +volume_backend_name = vitastor-testcluster +image_volume_cache_enabled = True +volume_clear = none +vitastor_etcd_address = 192.168.7.2:2379 +vitastor_etcd_prefix = +vitastor_config_path = /etc/vitastor/vitastor.conf +vitastor_pool_id = 1 +image_upload_use_cinder_backend = True +``` + +To put Glance images in Vitastor, use [https://docs.openstack.org/cinder/pike/admin/blockstorage-volume-backed-image.html](volume-backed images), +although the support has not been verified yet. + +### Proxmox + +To enable Vitastor support in Proxmox Virtual Environment (6.4 and 7.1 are supported): + +- Add the corresponding Vitastor Debian repository into sources.list on Proxmox hosts + (buster for 6.4, bullseye for 7.1) +- Install vitastor-client and pve-qemu-kvm from Vitastor repository +- Copy [patches/PVE_VitastorPlugin.pm](patches/PVE_VitastorPlugin.pm) to Proxmox hosts + as `/usr/share/perl5/PVE/Storage/Custom/VitastorPlugin.pm` +- Define storage in `/etc/pve/storage.cfg` (see below) +- Restart pvedaemon: `systemctl restart pvedaemon` + +`/etc/pve/storage.cfg` example (the only required option is vitastor_pool, all others +are listed below with their default values): + +``` +vitastor: vitastor + # pool to put new images into + vitastor_pool testpool + # path to the configuration file + vitastor_config_path /etc/vitastor/vitastor.conf + # etcd address(es), required only if missing in the configuration file + vitastor_etcd_address 192.168.7.2:2379/v3 + # prefix for keys in etcd + vitastor_etcd_prefix /vitastor + # prefix for images + vitastor_prefix pve/ + # use NBD mounter (only required for containers) + vitastor_nbd 0 +``` + ## Known Problems - Object deletion requests may currently lead to 'incomplete' objects in EC pools diff --git a/patches/PVE_VitastorPlugin.pm b/patches/PVE_VitastorPlugin.pm new file mode 100644 index 000000000..32e39e83b --- /dev/null +++ b/patches/PVE_VitastorPlugin.pm @@ -0,0 +1,485 @@ +# Install as /usr/share/perl5/PVE/Storage/Custom/VitastorPlugin.pm + +# Proxmox Vitastor Driver +# Copyright (c) Vitaliy Filippov, 2021+ +# License: VNPL-1.1 or GNU AGPLv3.0 + +package PVE::Storage::Custom::VitastorPlugin; + +use strict; +use warnings; + +use JSON; + +use PVE::Storage::Plugin; +use PVE::Tools qw(run_command); + +use base qw(PVE::Storage::Plugin); + +sub api +{ + # Trick it :) + return PVE::Storage->APIVER; +} + +sub run_cli +{ + my ($scfg, $cmd, %args) = @_; + my $retval; + my $stderr = ''; + my $errmsg = $args{errmsg} ? $args{errmsg}.": " : "vitastor-cli error: "; + my $json = delete $args{json}; + $json = 1 if !defined $json; + my $binary = delete $args{binary}; + $binary = '/usr/bin/vitastor-cli' if !defined $binary; + if (!exists($args{errfunc})) + { + $args{errfunc} = sub + { + my $line = shift; + print STDERR $line; + *STDERR->flush(); + $stderr .= $line; + }; + } + if (!exists($args{outfunc})) + { + $retval = ''; + $args{outfunc} = sub { $retval .= shift }; + if ($json) + { + unshift @$cmd, '--json'; + } + } + if ($scfg->{vitastor_etcd_address}) + { + unshift @$cmd, '--etcd_address', $scfg->{vitastor_etcd_address}; + } + if ($scfg->{vitastor_config_path}) + { + unshift @$cmd, '--config_path', $scfg->{vitastor_config_path}; + } + unshift @$cmd, $binary; + eval { run_command($cmd, %args); }; + if (my $err = $@) + { + die "Error invoking vitastor-cli: $err"; + } + if (defined $retval) + { + # untaint + $retval =~ /^(.*)$/s; + if ($json) + { + eval { $retval = JSON::decode_json($1); }; + if ($@) + { + die "vitastor-cli returned bad JSON: $@"; + } + } + else + { + $retval = $1; + } + } + return $retval; +} + +# Configuration + +sub type +{ + return 'vitastor'; +} + +sub plugindata +{ + return { + content => [ { images => 1, rootdir => 1 }, { images => 1 } ], + }; +} + +sub properties +{ + return { + vitastor_etcd_address => { + description => 'IP address(es) of etcd.', + type => 'string', + format => 'pve-storage-portal-dns-list', + }, + vitastor_etcd_prefix => { + description => 'Prefix for Vitastor etcd metadata', + type => 'string', + }, + vitastor_config_path => { + description => 'Path to Vitastor configuration file', + type => 'string', + }, + vitastor_prefix => { + description => 'Image name prefix', + type => 'string', + }, + vitastor_pool => { + description => 'Default pool to use for images', + type => 'string', + }, + vitastor_nbd => { + description => 'Use kernel NBD devices (slower)', + type => 'boolean', + }, + }; +} + +sub options +{ + return { + nodes => { optional => 1 }, + disable => { optional => 1 }, + vitastor_etcd_address => { optional => 1}, + vitastor_etcd_prefix => { optional => 1 }, + vitastor_config_path => { optional => 1 }, + vitastor_prefix => { optional => 1 }, + vitastor_pool => {}, + vitastor_nbd => { optional => 1 }, + }; +} + +# Storage implementation + +sub parse_volname +{ + my ($class, $volname) = @_; + if ($volname =~ m/^((base-(\d+)-\S+)\/)?((?:(base)|(vm))-(\d+)-\S+)$/) + { + # ($vtype, $name, $vmid, $basename, $basevmid, $isBase, $format) + return ('images', $4, $7, $2, $3, $5, 'raw'); + } + die "unable to parse vitastor volume name '$volname'\n"; +} + +sub _qemu_option +{ + my ($k, $v) = @_; + if (defined $v && $v ne "") + { + $v =~ s/:/\\:/gso; + return ":$k=$v"; + } + return ""; +} + +sub path +{ + my ($class, $scfg, $volname, $storeid, $snapname) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + my ($vtype, $name, $vmid) = $class->parse_volname($volname); + $name .= '@'.$snapname if $snapname; + if ($scfg->{vitastor_nbd}) + { + my $mapped = run_cli($scfg, [ 'ls' ], binary => '/usr/bin/vitastor-nbd'); + my ($kerneldev) = grep { $mapped->{$_}->{image} eq $prefix.$name } keys %$mapped; + die "Image not mapped via NBD" if !$kerneldev; + return ($kerneldev, $vmid, $vtype); + } + my $path = "vitastor"; + $path .= _qemu_option('config_path', $scfg->{vitastor_config_path}); + # FIXME This is the only exception: etcd_address -> etcd_host for qemu + $path .= _qemu_option('etcd_host', $scfg->{vitastor_etcd_address}); + $path .= _qemu_option('etcd_prefix', $scfg->{vitastor_etcd_prefix}); + $path .= _qemu_option('image', $prefix.$name); + return ($path, $vmid, $vtype); +} + +sub _find_free_diskname +{ + my ($class, $storeid, $scfg, $vmid, $fmt, $add_fmt_suffix) = @_; + my $list = _process_list($scfg, $storeid, run_cli($scfg, [ 'ls' ])); + $list = [ map { $_->{name} } @$list ]; + return PVE::Storage::Plugin::get_next_vm_diskname($list, $storeid, $vmid, undef, $scfg); +} + +# Used only in "Create Template" and, in fact, converts a VM into a template +# As a consequence, this is always invoked with the VM powered off +# So we just rename vm-xxx to base-xxx and make it a readonly base layer +sub create_base +{ + my ($class, $storeid, $scfg, $volname) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + + my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) = $class->parse_volname($volname); + die "create_base not possible with base image\n" if $isBase; + + my $info = _process_list($scfg, $storeid, run_cli($scfg, [ 'ls', $prefix.$name ]))->[0]; + die "image $name does not exist\n" if !$info; + + die "volname '$volname' contains wrong information about parent {$info->{parent}} $basename\n" + if $basename && (!$info->{parent} || $info->{parent} ne $basename); + + my $newname = $name; + $newname =~ s/^vm-/base-/; + + my $newvolname = $basename ? "$basename/$newname" : "$newname"; + run_cli($scfg, [ 'modify', '--rename', $prefix.$newname, '--readonly', $prefix.$name ], json => 0); + + return $newvolname; +} + +sub clone_image +{ + my ($class, $scfg, $storeid, $volname, $vmid, $snapname) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + + my $snap = ''; + $snap = '@'.$snapname if length $snapname; + + my ($vtype, $basename, $basevmid, undef, undef, $isBase) = $class->parse_volname($volname); + die "$volname is not a base image and snapname is not provided\n" if !$isBase && !length($snapname); + + my $name = $class->find_free_diskname($storeid, $scfg, $vmid); + + warn "clone $volname: $basename snapname $snap to $name\n"; + + my $newvol = "$basename/$name"; + $newvol = $name if length($snapname); + + run_cli($scfg, [ 'create', '--parent', $prefix.$basename.$snap, $prefix.$name ], json => 0); + + return $newvol; +} + +sub alloc_image +{ + my ($class, $storeid, $scfg, $vmid, $fmt, $name, $size) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + die "illegal name '$name' - should be 'vm-$vmid-*'\n" if $name && $name !~ m/^vm-$vmid-/; + $name = $class->find_free_diskname($storeid, $scfg, $vmid) if !$name; + run_cli($scfg, [ 'create', '--size', (int(($size+3)/4)*4).'k', '--pool', $scfg->{vitastor_pool}, $prefix.$name ], json => 0); + return $name; +} + +sub free_image +{ + my ($class, $storeid, $scfg, $volname, $isBase) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + my ($vtype, $name, $vmid, undef, undef, undef) = $class->parse_volname($volname); + $class->deactivate_volume($storeid, $scfg, $volname); + my $full_list = run_cli($scfg, [ 'ls', '-l' ]); + my $list = _process_list($scfg, $storeid, $full_list); + # Remove image and all its snapshots + my $to_remove = [ grep { $_->{name} eq $name || substr($_->{name}, 0, length($name)+1) eq ($name.'@') } @$list ]; + my $rm_names = { map { ($prefix.$_->{name} => 1) } @$to_remove }; + my $children = [ grep { $rm_names->{$_->{parent_name}} } @$full_list ]; + die "Image has children: ".join(', ', map { + substr($_->{name}, 0, length $prefix) eq $prefix + ? substr($_->name, length $prefix) + : $_->{name} + } @$children)."\n"; + for my $rmi (@$to_remove) + { + run_cli($scfg, [ 'rm-data', '--pool', $rmi->{pool_id}, '--inode', $rmi->{inode_num} ], json => 0); + } + for my $rmi (@$to_remove) + { + run_cli($scfg, [ 'rm', $prefix.$rmi->{name} ], json => 0); + } + return undef; +} + +sub _process_list +{ + my ($scfg, $storeid, $result) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + my $list = []; + foreach my $el (@$result) + { + next if !$el->{name} || length($prefix) && substr($el->{name}, 0, length $prefix) ne $prefix; + my $name = substr($el->{name}, length $prefix); + next if $name =~ /@/; + my ($owner) = $name =~ /^(?:vm|base)-(\d+)-/s; + next if !defined $owner; + my $parent = $prefix eq '' || substr($el->{parent_name}, 0, length $prefix) eq $prefix + ? substr($el->{parent_name}, length $prefix) : ''; + my $volid = $parent && $parent =~ /^(base-\d+-\S+)$/s + ? "$storeid:$1/$name" : "$storeid:$name"; + push @$list, { + format => 'raw', + volid => $volid, + name => $name, + size => $el->{size}, + parent => $parent, + vmid => $owner, + }; + } + return $list; +} + +sub list_images +{ + my ($class, $storeid, $scfg, $vmid, $vollist, $cache) = @_; + return _process_list($scfg, $storeid, run_cli($scfg, [ 'ls', '-l' ])); +} + +sub status +{ + my ($class, $storeid, $scfg, $cache) = @_; + # FIXME: take it from etcd + my $free = 0; + my $used = 0; + my $total = $used + $free; + my $active = 1; + return ($total, $free, $used, $active); +} + +sub activate_storage +{ + my ($class, $storeid, $scfg, $cache) = @_; + return 1; +} + +sub deactivate_storage +{ + my ($class, $storeid, $scfg, $cache) = @_; + return 1; +} + +sub map_volume +{ + my ($class, $storeid, $scfg, $volname, $snapname) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + + my ($vtype, $img_name, $vmid) = $class->parse_volname($volname); + my $name = $img_name; + $name .= '@'.$snapname if $snapname; + + my $mapped = run_cli($scfg, [ 'ls' ], binary => '/usr/bin/vitastor-nbd'); + my ($kerneldev) = grep { $mapped->{$_}->{image} eq $prefix.$name } keys %$mapped; + return $kerneldev if $kerneldev && -b $kerneldev; # already mapped + + $kerneldev = run_cli($scfg, [ 'map', '--image', $prefix.$name ], binary => '/usr/bin/vitastor-nbd', json => 0); + return $kerneldev; +} + +sub unmap_volume +{ + my ($class, $storeid, $scfg, $volname, $snapname) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + + return 1 if !$scfg->{vitastor_nbd}; + + my ($vtype, $name, $vmid) = $class->parse_volname($volname); + $name .= '@'.$snapname if $snapname; + + my $mapped = run_cli($scfg, [ 'ls' ], binary => '/usr/bin/vitastor-nbd'); + my ($kerneldev) = grep { $mapped->{$_}->{image} eq $prefix.$name } keys %$mapped; + if ($kerneldev && -b $kerneldev) + { + run_cli($scfg, [ 'unmap', $kerneldev ], binary => '/usr/bin/vitastor-nbd', json => 0); + } + + return 1; +} + +sub activate_volume +{ + my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_; + $class->map_volume($storeid, $scfg, $volname, $snapname) if $scfg->{vitastor_nbd}; + return 1; +} + +sub deactivate_volume +{ + my ($class, $storeid, $scfg, $volname, $snapname, $cache) = @_; + $class->unmap_volume($storeid, $scfg, $volname, $snapname); + return 1; +} + +sub volume_size_info +{ + my ($class, $scfg, $storeid, $volname, $timeout) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + my ($vtype, $name, $vmid) = $class->parse_volname($volname); + my $info = _process_list($scfg, $storeid, run_cli($scfg, [ 'ls', $prefix.$name ]))->[0]; + #return wantarray ? ($size, $format, $used, $parent, $st->ctime) : $size; + return $info->{size}; +} + +sub volume_resize +{ + my ($class, $scfg, $storeid, $volname, $size, $running) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + my ($vtype, $name, $vmid) = $class->parse_volname($volname); + run_cli($scfg, [ 'modify', '--resize', (int(($size+3)/4)*4).'k', $prefix.$name ], json => 0); + return undef; +} + +sub volume_snapshot +{ + my ($class, $scfg, $storeid, $volname, $snap) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + my ($vtype, $name, $vmid) = $class->parse_volname($volname); + run_cli($scfg, [ 'create', '--snapshot', $snap, $prefix.$name ], json => 0); + return undef; +} + +sub volume_snapshot_rollback +{ + my ($class, $scfg, $storeid, $volname, $snap) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + my ($vtype, $name, $vmid) = $class->parse_volname($volname); + run_cli($scfg, [ 'rm', $prefix.$name ], json => 0); + run_cli($scfg, [ 'create', '--parent', $prefix.$name.'@'.$snap, $prefix.$name ], json => 0); + return undef; +} + +sub volume_snapshot_delete +{ + my ($class, $scfg, $storeid, $volname, $snap, $running) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + my ($vtype, $name, $vmid) = $class->parse_volname($volname); + run_cli($scfg, [ 'rm', $prefix.$name.'@'.$snap ], json => 0); + return undef; +} + +sub volume_snapshot_needs_fsfreeze +{ + return 1; +} + +sub volume_has_feature +{ + my ($class, $scfg, $feature, $storeid, $volname, $snapname, $running) = @_; + my $features = { + snapshot => { current => 1, snap => 1 }, + clone => { base => 1, snap => 1 }, + template => { current => 1 }, + copy => { base => 1, current => 1, snap => 1 }, + sparseinit => { base => 1, current => 1 }, + rename => { current => 1 }, + }; + my ($vtype, $name, $vmid, $basename, $basevmid, $isBase) = $class->parse_volname($volname); + my $key = undef; + if ($snapname) + { + $key = 'snap'; + } + else + { + $key = $isBase ? 'base' : 'current'; + } + return 1 if $features->{$feature}->{$key}; + return undef; +} + +sub rename_volume +{ + my ($class, $scfg, $storeid, $source_volname, $target_vmid, $target_volname) = @_; + my $prefix = defined $scfg->{vitastor_prefix} ? $scfg->{vitastor_prefix} : 'pve/'; + my (undef, $source_image, $source_vmid, $base_name, $base_vmid, undef, $format) = + $class->parse_volname($source_volname); + $target_volname = $class->find_free_diskname($storeid, $scfg, $target_vmid, $format) if !$target_volname; + run_cli($scfg, [ 'modify', '--rename', $prefix.$target_volname, $prefix.$source_image ], json => 0); + $base_name = $base_name ? "${base_name}/" : ''; + return "${storeid}:${base_name}${target_volname}"; +} + +1; diff --git a/patches/nova-21.diff b/patches/nova-21.diff new file mode 100644 index 000000000..742cba049 --- /dev/null +++ b/patches/nova-21.diff @@ -0,0 +1,288 @@ +diff --git a/nova/virt/image/model.py b/nova/virt/image/model.py +index 971f7e9c07..ec3fca72cb 100644 +--- a/nova/virt/image/model.py ++++ b/nova/virt/image/model.py +@@ -129,3 +129,22 @@ class RBDImage(Image): + self.user = user + self.password = password + self.servers = servers ++ ++ ++class VitastorImage(Image): ++ """Class for images in a remote Vitastor cluster""" ++ ++ def __init__(self, name, etcd_address = None, etcd_prefix = None, config_path = None): ++ """Create a new Vitastor image object ++ ++ :param name: name of the image ++ :param etcd_address: etcd URL(s) (optional) ++ :param etcd_prefix: etcd prefix (optional) ++ :param config_path: path to the configuration (optional) ++ """ ++ super(VitastorImage, self).__init__(FORMAT_RAW) ++ ++ self.name = name ++ self.etcd_address = etcd_address ++ self.etcd_prefix = etcd_prefix ++ self.config_path = config_path +diff --git a/nova/virt/images.py b/nova/virt/images.py +index 5358f3766a..ebe3d6effb 100644 +--- a/nova/virt/images.py ++++ b/nova/virt/images.py +@@ -41,7 +41,7 @@ IMAGE_API = glance.API() + + def qemu_img_info(path, format=None): + """Return an object containing the parsed output from qemu-img info.""" +- if not os.path.exists(path) and not path.startswith('rbd:'): ++ if not os.path.exists(path) and not path.startswith('rbd:') and not path.startswith('vitastor:'): + raise exception.DiskNotFound(location=path) + + info = nova.privsep.qemu.unprivileged_qemu_img_info(path, format=format) +@@ -50,7 +50,7 @@ def qemu_img_info(path, format=None): + + def privileged_qemu_img_info(path, format=None, output_format='json'): + """Return an object containing the parsed output from qemu-img info.""" +- if not os.path.exists(path) and not path.startswith('rbd:'): ++ if not os.path.exists(path) and not path.startswith('rbd:') and not path.startswith('vitastor:'): + raise exception.DiskNotFound(location=path) + + info = nova.privsep.qemu.privileged_qemu_img_info(path, format=format) +diff --git a/nova/virt/libvirt/config.py b/nova/virt/libvirt/config.py +index ea525648b3..d7aa798954 100644 +--- a/nova/virt/libvirt/config.py ++++ b/nova/virt/libvirt/config.py +@@ -1005,6 +1005,8 @@ class LibvirtConfigGuestDisk(LibvirtConfigGuestDevice): + self.driver_iommu = False + self.source_path = None + self.source_protocol = None ++ self.source_query = None ++ self.source_config = None + self.source_name = None + self.source_hosts = [] + self.source_ports = [] +@@ -1133,6 +1135,10 @@ class LibvirtConfigGuestDisk(LibvirtConfigGuestDevice): + source = etree.Element("source", protocol=self.source_protocol) + if self.source_name is not None: + source.set('name', self.source_name) ++ if self.source_query is not None: ++ source.set('query', self.source_query) ++ if self.source_config is not None: ++ source.append(etree.Element('config', file=self.source_config)) + hosts_info = zip(self.source_hosts, self.source_ports) + for name, port in hosts_info: + host = etree.Element('host', name=name) +diff --git a/nova/virt/libvirt/driver.py b/nova/virt/libvirt/driver.py +index fbd033690a..74dc59ce87 100644 +--- a/nova/virt/libvirt/driver.py ++++ b/nova/virt/libvirt/driver.py +@@ -180,6 +180,7 @@ libvirt_volume_drivers = [ + 'local=nova.virt.libvirt.volume.volume.LibvirtVolumeDriver', + 'fake=nova.virt.libvirt.volume.volume.LibvirtFakeVolumeDriver', + 'rbd=nova.virt.libvirt.volume.net.LibvirtNetVolumeDriver', ++ 'vitastor=nova.virt.libvirt.volume.vitastor.LibvirtVitastorVolumeDriver', + 'nfs=nova.virt.libvirt.volume.nfs.LibvirtNFSVolumeDriver', + 'smbfs=nova.virt.libvirt.volume.smbfs.LibvirtSMBFSVolumeDriver', + 'fibre_channel=' +@@ -287,10 +288,10 @@ class LibvirtDriver(driver.ComputeDriver): + # This prevents the risk of one test setting a capability + # which bleeds over into other tests. + +- # LVM and RBD require raw images. If we are not configured to ++ # LVM, RBD, Vitastor require raw images. If we are not configured to + # force convert images into raw format, then we _require_ raw + # images only. +- raw_only = ('rbd', 'lvm') ++ raw_only = ('rbd', 'lvm', 'vitastor') + requires_raw_image = (CONF.libvirt.images_type in raw_only and + not CONF.force_raw_images) + requires_ploop_image = CONF.libvirt.virt_type == 'parallels' +@@ -703,12 +704,12 @@ class LibvirtDriver(driver.ComputeDriver): + # Some imagebackends are only able to import raw disk images, + # and will fail if given any other format. See the bug + # https://bugs.launchpad.net/nova/+bug/1816686 for more details. +- if CONF.libvirt.images_type in ('rbd',): ++ if CONF.libvirt.images_type in ('rbd', 'vitastor'): + if not CONF.force_raw_images: + msg = _("'[DEFAULT]/force_raw_images = False' is not " +- "allowed with '[libvirt]/images_type = rbd'. " ++ "allowed with '[libvirt]/images_type = rbd' or 'vitastor'. " + "Please check the two configs and if you really " +- "do want to use rbd as images_type, set " ++ "do want to use rbd or vitastor as images_type, set " + "force_raw_images to True.") + raise exception.InvalidConfiguration(msg) + +@@ -2165,6 +2166,16 @@ class LibvirtDriver(driver.ComputeDriver): + if connection_info['data'].get('auth_enabled'): + username = connection_info['data']['auth_username'] + path = f"rbd:{volume_name}:id={username}" ++ elif connection_info['driver_volume_type'] == 'vitastor': ++ volume_name = connection_info['data']['name'] ++ path = 'vitastor:image='+volume_name.replace(':', '\\:') ++ for k in [ 'config_path', 'etcd_address', 'etcd_prefix' ]: ++ if k in connection_info['data']: ++ kk = k ++ if kk == 'etcd_address': ++ # FIXME use etcd_address in qemu driver ++ kk = 'etcd_host' ++ path += ":"+kk.replace('_', '-')+"="+connection_info['data'][k].replace(':', '\\:') + else: + path = 'unknown' + raise exception.DiskNotFound(location='unknown') +@@ -2440,8 +2451,8 @@ class LibvirtDriver(driver.ComputeDriver): + + image_format = CONF.libvirt.snapshot_image_format or source_type + +- # NOTE(bfilippov): save lvm and rbd as raw +- if image_format == 'lvm' or image_format == 'rbd': ++ # NOTE(bfilippov): save lvm and rbd and vitastor as raw ++ if image_format == 'lvm' or image_format == 'rbd' or image_format == 'vitastor': + image_format = 'raw' + + metadata = self._create_snapshot_metadata(instance.image_meta, +@@ -2512,7 +2523,7 @@ class LibvirtDriver(driver.ComputeDriver): + expected_state=task_states.IMAGE_UPLOADING) + + # TODO(nic): possibly abstract this out to the root_disk +- if source_type == 'rbd' and live_snapshot: ++ if (source_type == 'rbd' or source_type == 'vitastor') and live_snapshot: + # Standard snapshot uses qemu-img convert from RBD which is + # not safe to run with live_snapshot. + live_snapshot = False +@@ -3715,7 +3726,7 @@ class LibvirtDriver(driver.ComputeDriver): + # cleanup rescue volume + lvm.remove_volumes([lvmdisk for lvmdisk in self._lvm_disks(instance) + if lvmdisk.endswith('.rescue')]) +- if CONF.libvirt.images_type == 'rbd': ++ if CONF.libvirt.images_type == 'rbd' or CONF.libvirt.images_type == 'vitastor': + filter_fn = lambda disk: (disk.startswith(instance.uuid) and + disk.endswith('.rescue')) + rbd_utils.RBDDriver().cleanup_volumes(filter_fn) +@@ -3972,6 +3983,8 @@ class LibvirtDriver(driver.ComputeDriver): + # TODO(mikal): there is a bug here if images_type has + # changed since creation of the instance, but I am pretty + # sure that this bug already exists. ++ if CONF.libvirt.images_type == 'vitastor': ++ return 'vitastor' + return 'rbd' if CONF.libvirt.images_type == 'rbd' else 'raw' + + @staticmethod +@@ -4370,10 +4383,10 @@ class LibvirtDriver(driver.ComputeDriver): + finally: + # NOTE(mikal): if the config drive was imported into RBD, + # then we no longer need the local copy +- if CONF.libvirt.images_type == 'rbd': ++ if CONF.libvirt.images_type == 'rbd' or CONF.libvirt.images_type == 'vitastor': + LOG.info('Deleting local config drive %(path)s ' +- 'because it was imported into RBD.', +- {'path': config_disk_local_path}, ++ 'because it was imported into %(type).', ++ {'path': config_disk_local_path, 'type': CONF.libvirt.images_type}, + instance=instance) + os.unlink(config_disk_local_path) + +diff --git a/nova/virt/libvirt/utils.py b/nova/virt/libvirt/utils.py +index c1dc34daf4..263965912f 100644 +--- a/nova/virt/libvirt/utils.py ++++ b/nova/virt/libvirt/utils.py +@@ -399,6 +399,10 @@ def find_disk(guest: libvirt_guest.Guest) -> ty.Tuple[str, ty.Optional[str]]: + disk_path = disk.source_name + if disk_path: + disk_path = 'rbd:' + disk_path ++ elif not disk_path and disk.source_protocol == 'vitastor': ++ disk_path = disk.source_name ++ if disk_path: ++ disk_path = 'vitastor:' + disk_path + + if not disk_path: + raise RuntimeError(_("Can't retrieve root device path " +@@ -417,6 +421,8 @@ def get_disk_type_from_path(path: str) -> ty.Optional[str]: + return 'lvm' + elif path.startswith('rbd:'): + return 'rbd' ++ elif path.startswith('vitastor:'): ++ return 'vitastor' + elif (os.path.isdir(path) and + os.path.exists(os.path.join(path, "DiskDescriptor.xml"))): + return 'ploop' +diff --git a/nova/virt/libvirt/volume/vitastor.py b/nova/virt/libvirt/volume/vitastor.py +new file mode 100644 +index 0000000000..0256df62c1 +--- /dev/null ++++ b/nova/virt/libvirt/volume/vitastor.py +@@ -0,0 +1,75 @@ ++# Copyright (c) 2021+, Vitaliy Filippov ++# ++# Licensed under the Apache License, Version 2.0 (the "License"); you may ++# not use this file except in compliance with the License. You may obtain ++# a copy of the License at ++# ++# http://www.apache.org/licenses/LICENSE-2.0 ++# ++# Unless required by applicable law or agreed to in writing, software ++# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT ++# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the ++# License for the specific language governing permissions and limitations ++# under the License. ++ ++from os_brick import exception as os_brick_exception ++from os_brick import initiator ++from os_brick.initiator import connector ++from oslo_log import log as logging ++ ++import nova.conf ++from nova import utils ++from nova.virt.libvirt.volume import volume as libvirt_volume ++ ++ ++CONF = nova.conf.CONF ++LOG = logging.getLogger(__name__) ++ ++ ++class LibvirtVitastorVolumeDriver(libvirt_volume.LibvirtBaseVolumeDriver): ++ """Driver to attach Vitastor volumes to libvirt.""" ++ def __init__(self, host): ++ super(LibvirtVitastorVolumeDriver, self).__init__(host, is_block_dev=False) ++ ++ def connect_volume(self, connection_info, instance): ++ pass ++ ++ def disconnect_volume(self, connection_info, instance): ++ pass ++ ++ def get_config(self, connection_info, disk_info): ++ """Returns xml for libvirt.""" ++ conf = super(LibvirtVitastorVolumeDriver, self).get_config(connection_info, disk_info) ++ conf.source_type = 'network' ++ conf.source_protocol = 'vitastor' ++ conf.source_name = connection_info['data'].get('name') ++ conf.source_query = connection_info['data'].get('etcd_prefix') or None ++ conf.source_config = connection_info['data'].get('config_path') or None ++ conf.source_hosts = [] ++ conf.source_ports = [] ++ addresses = connection_info['data'].get('etcd_address', '') ++ if addresses: ++ if not isinstance(addresses, list): ++ addresses = addresses.split(',') ++ for addr in addresses: ++ if addr.startswith('https://'): ++ raise NotImplementedError('Vitastor block driver does not support SSL for etcd communication yet') ++ if addr.startswith('http://'): ++ addr = addr[7:] ++ addr = addr.rstrip('/') ++ if addr.endswith('/v3'): ++ addr = addr[0:-3] ++ p = addr.find('/') ++ if p > 0: ++ raise NotImplementedError('libvirt does not support custom URL paths for Vitastor etcd yet. Use /etc/vitastor/vitastor.conf') ++ p = addr.find(':') ++ port = '2379' ++ if p > 0: ++ port = addr[p+1:] ++ addr = addr[0:p] ++ conf.source_hosts.append(addr) ++ conf.source_ports.append(port) ++ return conf ++ ++ def extend_volume(self, connection_info, instance, requested_size): ++ raise NotImplementedError