- ability to mount through kernel and fuse client - allow mount options - get MONs from ceph config if not in storage.cfg - allow the use of ceph config with fuse client
Signed-off-by: Alwin Antreich <a.antre...@proxmox.com> --- PVE/API2/Storage/Config.pm | 28 ++----- PVE/Storage.pm | 2 + PVE/Storage/CephFSPlugin.pm | 177 ++++++++++++++++++++++++++++++++++++++++++++ PVE/Storage/CephTools.pm | 102 +++++++++++++++++++++++++ PVE/Storage/Makefile | 2 +- PVE/Storage/Plugin.pm | 1 + debian/control | 1 + 7 files changed, 292 insertions(+), 21 deletions(-) create mode 100644 PVE/Storage/CephFSPlugin.pm diff --git a/PVE/API2/Storage/Config.pm b/PVE/API2/Storage/Config.pm index 258820d..3abd6e8 100755 --- a/PVE/API2/Storage/Config.pm +++ b/PVE/API2/Storage/Config.pm @@ -10,6 +10,7 @@ use PVE::Storage; use PVE::Storage::Plugin; use PVE::Storage::LVMPlugin; use PVE::Storage::CIFSPlugin; +use PVE::Storage::CephTools; use HTTP::Status qw(:constants); use Storable qw(dclone); use PVE::JSONSchema qw(get_standard_option); @@ -171,21 +172,10 @@ __PACKAGE__->register_method ({ PVE::Storage::activate_storage($cfg, $baseid); PVE::Storage::LVMPlugin::lvm_create_volume_group($path, $opts->{vgname}, $opts->{shared}); - } elsif ($type eq 'rbd' && !defined($opts->{monhost})) { - my $ceph_admin_keyring = '/etc/pve/priv/ceph.client.admin.keyring'; - my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.keyring"; - - die "ceph authx keyring file for storage '$storeid' already exists!\n" - if -e $ceph_storage_keyring; - - eval { - mkdir '/etc/pve/priv/ceph'; - PVE::Tools::file_copy($ceph_admin_keyring, $ceph_storage_keyring); - }; - if (my $err = $@) { - unlink $ceph_storage_keyring; - die "failed to copy ceph authx keyring for storage '$storeid': $err\n"; - } + + } elsif (($type eq 'rbd' || $type eq 'cephfs') && !defined($opts->{monhost})) { + # create keyring or secret file in /etc/pve/priv/ceph/ + PVE::Storage::CephTools::ceph_create_keyfile($type, $storeid); } # create a password file in /etc/pve/priv, # this file is used as a cert_file at mount time. @@ -289,11 +279,9 @@ __PACKAGE__->register_method ({ if (-f $cred_file) { unlink($cred_file) or warn "removing cifs credientials '$cred_file' failed: $!\n"; } - } elsif ($scfg->{type} eq 'rbd' && !defined($scfg->{monhost})) { - my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.keyring"; - if (-f $ceph_storage_keyring) { - unlink($ceph_storage_keyring) or warn "removing keyring of storage failed: $!\n"; - } + } elsif (($scfg->{type} eq 'rbd' || $scfg->{type} eq 'cephfs') && !defined($scfg->{monhost})) { + # remove keyring or secret file in /etc/pve/priv/ceph/ + PVE::Storage::CephTools::ceph_remove_keyfile($scfg->{type}, $storeid); } delete $cfg->{ids}->{$storeid}; diff --git a/PVE/Storage.pm b/PVE/Storage.pm index d733380..f9732fe 100755 --- a/PVE/Storage.pm +++ b/PVE/Storage.pm @@ -28,6 +28,7 @@ use PVE::Storage::NFSPlugin; use PVE::Storage::CIFSPlugin; use PVE::Storage::ISCSIPlugin; use PVE::Storage::RBDPlugin; +use PVE::Storage::CephFSPlugin; use PVE::Storage::SheepdogPlugin; use PVE::Storage::ISCSIDirectPlugin; use PVE::Storage::GlusterfsPlugin; @@ -46,6 +47,7 @@ PVE::Storage::NFSPlugin->register(); PVE::Storage::CIFSPlugin->register(); PVE::Storage::ISCSIPlugin->register(); PVE::Storage::RBDPlugin->register(); +PVE::Storage::CephFSPlugin->register(); PVE::Storage::SheepdogPlugin->register(); PVE::Storage::ISCSIDirectPlugin->register(); PVE::Storage::GlusterfsPlugin->register(); diff --git a/PVE/Storage/CephFSPlugin.pm b/PVE/Storage/CephFSPlugin.pm new file mode 100644 index 0000000..fa687f4 --- /dev/null +++ b/PVE/Storage/CephFSPlugin.pm @@ -0,0 +1,177 @@ +package PVE::Storage::CephFSPlugin; + +use strict; +use warnings; +use IO::File; +use Net::IP; +use File::Path; +use PVE::Tools qw(run_command); +use PVE::ProcFSTools; +use PVE::Storage::Plugin; +use PVE::JSONSchema qw(get_standard_option); +use PVE::Storage::CephTools; + +use base qw(PVE::Storage::Plugin); + +sub cephfs_is_mounted { + my ($scfg, $storeid, $mountdata) = @_; + + my $cmd_option = PVE::Storage::CephTools::ceph_connect_option($scfg, $storeid); + my $configfile = $cmd_option->{ceph_conf}; + my $server = $cmd_option->{mon_host} // PVE::Storage::CephTools::get_monaddr_list($configfile); + + my $subdir = $scfg->{subdir} // '/'; + my $mountpoint = $scfg->{path}; + my $source = "$server:$subdir"; + + $mountdata = PVE::ProcFSTools::parse_proc_mounts() if !$mountdata; + return $mountpoint if grep { + $_->[2] =~ m#^ceph|fuse\.ceph-fuse# && + $_->[0] =~ m#^\Q$source\E|ceph-fuse$# && + $_->[1] eq $mountpoint + } @$mountdata; + + warn "A filesystem is already mounted on $mountpoint\n" + if grep { $_->[1] eq $mountpoint } @$mountdata; + + return undef; +} + +sub cephfs_mount { + my ($scfg, $storeid) = @_; + + my $cmd; + my $mountpoint = $scfg->{path}; + my $subdir = $scfg->{subdir} // '/'; + + my $cmd_option = PVE::Storage::CephTools::ceph_connect_option($scfg, $storeid); + my $configfile = $cmd_option->{ceph_conf}; + my $secretfile = $cmd_option->{keyring}; + my $server = $cmd_option->{mon_host} // PVE::Storage::CephTools::get_monaddr_list($configfile); + + # fuse -> client-enforced quotas (kernel doesn't), updates w/ ceph-fuse pkg + # kernel -> better performance, less frequent updates + if ($scfg->{fuse}) { + # FIXME: ceph-fuse client complains about missing ceph.conf or keyring if + # not provided on its default locations but still connects. Fix upstream?? + $cmd = ['/usr/bin/ceph-fuse', '-n', "client.$cmd_option->{userid}", '-m', $server]; + push @$cmd, '--keyfile', $secretfile if defined($secretfile); + push @$cmd, '-r', $subdir if !($subdir =~ m|^/$|); + push @$cmd, $mountpoint; + push @$cmd, '--conf', $configfile if defined($configfile); + }else { + my $source = "$server:$subdir"; + $cmd = ['/bin/mount', '-t', 'ceph', $source, $mountpoint, '-o', "name=$cmd_option->{userid}"]; + push @$cmd, '-o', "secretfile=$secretfile" if defined($secretfile); + } + + if ($scfg->{options}) { + push @$cmd, '-o', $scfg->{options}; + } + + run_command($cmd, errmsg => "mount error"); +} + +# Configuration + +sub type { + return 'cephfs'; +} + +sub plugindata { + return { + content => [ { vztmpl => 1, iso => 1, backup => 1}, + { backup => 1 }], + }; +} + +sub properties { + return { + fuse => { + description => "Mount CephFS through FUSE.", + type => 'boolean', + }, + subdir => { + description => "Subdir to mount.", + type => 'string', format => 'pve-storage-path', + }, + }; +} + +sub options { + return { + path => { fixed => 1 }, + monhost => { optional => 1}, + nodes => { optional => 1 }, + subdir => { optional => 1 }, + disable => { optional => 1 }, + options => { optional => 1 }, + username => { optional => 1 }, + content => { optional => 1 }, + format => { optional => 1 }, + mkdir => { optional => 1 }, + fuse => { optional => 1 }, + bwlimit => { optional => 1 }, + }; +} + +sub check_config { + my ($class, $sectionId, $config, $create, $skipSchemaCheck) = @_; + + $config->{path} = "/mnt/pve/$sectionId" if $create && !$config->{path}; + + return $class->SUPER::check_config($sectionId, $config, $create, $skipSchemaCheck); +} + +# Storage implementation + +sub status { + my ($class, $storeid, $scfg, $cache) = @_; + + $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts() + if !$cache->{mountdata}; + + return undef if !cephfs_is_mounted($scfg, $storeid, $cache->{mountdata}); + + return $class->SUPER::status($storeid, $scfg, $cache); +} + +sub activate_storage { + my ($class, $storeid, $scfg, $cache) = @_; + + $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts() + if !$cache->{mountdata}; + + my $path = $scfg->{path}; + + if (!cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) { + + # NOTE: only call mkpath when not mounted (avoid hang + # when cephfs is offline + + mkpath $path if !(defined($scfg->{mkdir}) && !$scfg->{mkdir}); + + die "unable to activate storage '$storeid' - " . + "directory '$path' does not exist\n" if ! -d $path; + + cephfs_mount($scfg, $storeid); + } + + $class->SUPER::activate_storage($storeid, $scfg, $cache); +} + +sub deactivate_storage { + my ($class, $storeid, $scfg, $cache) = @_; + + $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts() + if !$cache->{mountdata}; + + my $path = $scfg->{path}; + + if (cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) { + my $cmd = ['/bin/umount', $path]; + run_command($cmd, errmsg => 'umount error'); + } +} + +1; diff --git a/PVE/Storage/CephTools.pm b/PVE/Storage/CephTools.pm index 3e2cede..766163d 100644 --- a/PVE/Storage/CephTools.pm +++ b/PVE/Storage/CephTools.pm @@ -34,6 +34,64 @@ my $ceph_check_keyfile = sub { return undef; }; +my $parse_ceph_file = sub { + my ($filename) = @_; + + my $cfg = {}; + + return $cfg if ! -f $filename; + + my $content = PVE::Tools::file_get_contents($filename); + my @lines = split /\n/, $content; + + my $section; + + foreach my $line (@lines) { + $line =~ s/[;#].*$//; + $line =~ s/^\s+//; + $line =~ s/\s+$//; + next if !$line; + + $section = $1 if $line =~ m/^\[(\S+)\]$/; + if (!$section) { + warn "no section - skip: $line\n"; + next; + } + + if ($line =~ m/^(.*?\S)\s*=\s*(\S.*)$/) { + $cfg->{$section}->{$1} = $2; + } + + } + + return $cfg; +}; + +my $ceph_get_key = sub { + my ($keyfile, $username) = @_; + + my $key = $parse_ceph_file->($keyfile); + my $secret = $key->{"client.$username"}->{key}; + + return $secret; +}; + +sub get_monaddr_list { + my ($configfile) = shift; + + my $server; + + if (!defined($configfile)) { + warn "No ceph config specified\n"; + return; + } + + my $config = $parse_ceph_file->($configfile); + @$server = sort map { $config->{$_}->{'mon addr'} } grep {/mon/} %{$config}; + + return join(',', @$server); +}; + sub hostlist { my ($list_text, $separator) = @_; @@ -85,4 +143,48 @@ sub ceph_connect_option { } +sub ceph_create_keyfile { + my ($type, $storeid) = @_; + + my $extension = 'keyring'; + $extension = 'secret' if ($type eq 'cephfs'); + + my $ceph_admin_keyring = '/etc/pve/priv/ceph.client.admin.keyring'; + my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.$extension"; + + die "ceph authx keyring file for storage '$storeid' already exists!\n" + if -e $ceph_storage_keyring; + + if (-e $ceph_admin_keyring) { + eval { + if ($type eq 'rbd') { + mkdir '/etc/pve/priv/ceph'; + PVE::Tools::file_copy($ceph_admin_keyring, $ceph_storage_keyring); + } elsif ($type eq 'cephfs') { + my $secret = $ceph_get_key->($ceph_admin_keyring, 'admin'); + mkdir '/etc/pve/priv/ceph'; + PVE::Tools::file_set_contents($ceph_storage_keyring, $secret, 0400); + } + }; + if (my $err = $@) { + unlink $ceph_storage_keyring; + die "failed to copy ceph authx $extension for storage '$storeid': $err\n"; + } + } else { + warn "$ceph_admin_keyring not found, authentication is disabled.\n"; + } +} + +sub ceph_remove_keyfile { + my ($type, $storeid) = @_; + + my $extension = 'keyring'; + $extension = 'secret' if ($type eq 'cephfs'); + my $ceph_storage_keyring = "/etc/pve/priv/ceph/${storeid}.$extension"; + + if (-f $ceph_storage_keyring) { + unlink($ceph_storage_keyring) or warn "removing keyring of storage failed: $!\n"; + } +} + 1; diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile index 82dadd6..c7f423f 100644 --- a/PVE/Storage/Makefile +++ b/PVE/Storage/Makefile @@ -1,4 +1,4 @@ -SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm ISCSIPlugin.pm RBDPlugin.pm CephTools.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm +SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm ISCSIPlugin.pm CephFSPlugin.pm RBDPlugin.pm CephTools.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm .PHONY: install install: diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm index 163871d..3769e01 100644 --- a/PVE/Storage/Plugin.pm +++ b/PVE/Storage/Plugin.pm @@ -24,6 +24,7 @@ our @SHARED_STORAGE = ( 'nfs', 'cifs', 'rbd', + 'cephfs', 'sheepdog', 'iscsidirect', 'glusterfs', diff --git a/debian/control b/debian/control index 908dd24..9923350 100644 --- a/debian/control +++ b/debian/control @@ -28,6 +28,7 @@ Depends: cstream, udev, smbclient, ceph-common, + ceph-fuse, cifs-utils, ${perl:Depends}, Description: Proxmox VE storage management library -- 2.11.0 _______________________________________________ pve-devel mailing list pve-devel@pve.proxmox.com https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel