- ability to mount through kernel and fuse client
 - allow mount options
 - get MONs from ceph config if not in storage.cfg
 - allow the use of ceph config with fuse client

Signed-off-by: Alwin Antreich <a.antre...@proxmox.com>
---
 PVE/API2/Storage/Config.pm  |   2 +-
 PVE/API2/Storage/Status.pm  |   2 +-
 PVE/Storage.pm              |   2 +
 PVE/Storage/CephFSPlugin.pm | 262 ++++++++++++++++++++++++++++++++++++++++++++
 PVE/Storage/Makefile        |   2 +-
 PVE/Storage/Plugin.pm       |   1 +
 debian/control              |   2 +
 7 files changed, 270 insertions(+), 3 deletions(-)
 create mode 100644 PVE/Storage/CephFSPlugin.pm

diff --git a/PVE/API2/Storage/Config.pm b/PVE/API2/Storage/Config.pm
index 3b38304..368a5c9 100755
--- a/PVE/API2/Storage/Config.pm
+++ b/PVE/API2/Storage/Config.pm
@@ -171,7 +171,7 @@ __PACKAGE__->register_method ({
                    PVE::Storage::activate_storage($cfg, $baseid);
 
                    PVE::Storage::LVMPlugin::lvm_create_volume_group($path, 
$opts->{vgname}, $opts->{shared});
-               } elsif ($type eq 'rbd' && !defined($opts->{monhost})) {
+               } elsif (($type eq 'rbd' || $type eq 'cephfs') && 
!defined($opts->{monhost})) {
                    my $ceph_admin_keyring = 
'/etc/pve/priv/ceph.client.admin.keyring';
                    my $ceph_storage_keyring = 
"/etc/pve/priv/ceph/${storeid}.keyring";
 
diff --git a/PVE/API2/Storage/Status.pm b/PVE/API2/Storage/Status.pm
index ab07146..2d8d143 100644
--- a/PVE/API2/Storage/Status.pm
+++ b/PVE/API2/Storage/Status.pm
@@ -335,7 +335,7 @@ __PACKAGE__->register_method ({
        my $scfg = PVE::Storage::storage_check_enabled($cfg, $param->{storage}, 
$node);
 
        die "cant upload to storage type '$scfg->{type}'\n" 
-           if !($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || 
$scfg->{type} eq 'glusterfs');
+           if !($scfg->{type} eq 'dir' || $scfg->{type} eq 'nfs' || 
$scfg->{type} eq 'glusterfs' || $scfg->{type} eq 'cephfs');
 
        my $content = $param->{content};
 
diff --git a/PVE/Storage.pm b/PVE/Storage.pm
index d733380..f9732fe 100755
--- a/PVE/Storage.pm
+++ b/PVE/Storage.pm
@@ -28,6 +28,7 @@ use PVE::Storage::NFSPlugin;
 use PVE::Storage::CIFSPlugin;
 use PVE::Storage::ISCSIPlugin;
 use PVE::Storage::RBDPlugin;
+use PVE::Storage::CephFSPlugin;
 use PVE::Storage::SheepdogPlugin;
 use PVE::Storage::ISCSIDirectPlugin;
 use PVE::Storage::GlusterfsPlugin;
@@ -46,6 +47,7 @@ PVE::Storage::NFSPlugin->register();
 PVE::Storage::CIFSPlugin->register();
 PVE::Storage::ISCSIPlugin->register();
 PVE::Storage::RBDPlugin->register();
+PVE::Storage::CephFSPlugin->register();
 PVE::Storage::SheepdogPlugin->register();
 PVE::Storage::ISCSIDirectPlugin->register();
 PVE::Storage::GlusterfsPlugin->register();
diff --git a/PVE/Storage/CephFSPlugin.pm b/PVE/Storage/CephFSPlugin.pm
new file mode 100644
index 0000000..a368c5b
--- /dev/null
+++ b/PVE/Storage/CephFSPlugin.pm
@@ -0,0 +1,262 @@
+package PVE::Storage::CephFSPlugin;
+
+use strict;
+use warnings;
+use IO::File;
+use Net::IP;
+use File::Path;
+use PVE::Tools qw(run_command);
+use PVE::ProcFSTools;
+use PVE::Storage::Plugin;
+use PVE::JSONSchema qw(get_standard_option);
+
+use base qw(PVE::Storage::Plugin);
+
+my $hostlist = sub {
+    my ($list_text, $separator) = @_;
+
+    my @monhostlist = PVE::Tools::split_list($list_text);
+    return join($separator, map {
+       my ($host, $port) = PVE::Tools::parse_host_and_port($_);
+       $port = defined($port) ? ":$port" : '';
+       $host = "[$host]" if Net::IP::ip_is_ipv6($host);
+       "${host}${port}"
+    } @monhostlist);
+};
+
+my $parse_ceph_config = sub {
+    my ($filename) = @_;
+
+    my $cfg = {};
+
+    return $cfg if ! -f $filename;
+
+    my $fh = IO::File->new($filename, "r") ||
+       die "unable to open '$filename' - $!\n";
+
+    my $section;
+
+    while (defined(my $line = <$fh>)) {
+       $line =~ s/[;#].*$//;
+       $line =~ s/^\s+//;
+       $line =~ s/\s+$//;
+       next if !$line;
+
+       $section = $1 if $line =~ m/^\[(\S+)\]$/;
+       if (!$section) {
+           warn "no section - skip: $line\n";
+           next;
+       }
+
+       if ($line =~ m/^(.*?\S)\s*=\s*(\S.*)$/) {
+           $cfg->{$section}->{$1} = $2;
+       }
+
+    }
+
+    return $cfg;
+};
+
+my $get_monaddr_list = sub {
+    my ($scfg, $configfile) = @_;
+
+    my $server;
+    my $no_mon = !defined($scfg->{monhost});
+
+    if (($no_mon) && defined($configfile)) {
+       my $config = $parse_ceph_config->($configfile);
+       $server = join(',', sort { $a cmp $b }
+           map { $config->{$_}->{'mon addr'} } grep {/mon/} %{$config});
+    }else {
+       $server = $hostlist->($scfg->{monhost}, ',');
+    }
+
+    return $server;
+};
+
+my $get_configfile = sub {
+    my ($storeid) = @_;
+
+    my $configfile;
+    my $pve_cephconfig = '/etc/pve/ceph.conf';
+    my $storeid_cephconfig = "/etc/pve/priv/ceph/${storeid}.conf";
+
+    if (-e $pve_cephconfig) {
+       if (-e $storeid_cephconfig) {
+           warn "ignoring custom ceph config for storage '$storeid', 'monhost' 
is not set (assuming pveceph managed cluster)!\n";
+       }
+       $configfile = $pve_cephconfig;
+    } elsif (-e $storeid_cephconfig) {
+       $configfile = $storeid_cephconfig;
+    } else {
+       die "Missing ceph config for ${storeid} storage\n";
+    }
+
+    return $configfile;
+};
+
+sub cephfs_is_mounted {
+    my ($scfg, $storeid, $mountdata) = @_;
+
+    my $no_mon = !defined($scfg->{monhost});
+
+    my $configfile = $get_configfile->($storeid) if ($no_mon);
+    my $server = $get_monaddr_list->($scfg, $configfile);
+
+    my $subdir = $scfg->{subdir} ? $scfg->{subdir} : '/';
+    my $mountpoint = $scfg->{path};
+    my $source = "$server:$subdir";
+
+    $mountdata = PVE::ProcFSTools::parse_proc_mounts() if !$mountdata;
+    return $mountpoint if grep {
+       $_->[2] =~ /^ceph|fuse\.ceph-fuse/ &&
+       $_->[0] =~ m#^\Q$source\E|ceph-fuse$# &&
+       $_->[1] eq $mountpoint
+    } @$mountdata;
+
+    warn "A filesystem is still mounted on $mountpoint\n"
+       if grep { $_->[1] eq $mountpoint } @$mountdata;
+
+    return undef;
+}
+
+sub cephfs_mount {
+    my ($scfg, $storeid) = @_;
+
+    my $cmd;
+
+    my $no_mon = !defined($scfg->{monhost});
+
+    my $keyfile = "/etc/pve/priv/ceph/${storeid}.keyring";
+    my $username =  $scfg->{username} ? $scfg->{username} : 'admin';
+    my $mountpoint = $scfg->{path};
+    my $subdir = $scfg->{subdir} ? $scfg->{subdir} : '/';
+
+    my $configfile = $get_configfile->($storeid) if ($no_mon);
+    my $server = $get_monaddr_list->($scfg, $configfile);
+
+    # fuse -> client-enforced quotas (kernel doesn't), updates w/ ceph-fuse pkg
+    # kernel -> better performance, less frequent updates
+    if ($scfg->{fuse}) {
+           # FIXME: ceph-fuse client complains about missing ceph.conf or 
keyring if
+           # not provided on its default locations but still connects. Fix 
upstream??
+           $cmd = ['/usr/bin/ceph-fuse', '-n', "client.$username", '-m', 
$server];
+           push @$cmd, '--keyfile', $keyfile if (-e $keyfile);
+           push @$cmd, '-r', $subdir if !($subdir =~ m|^/$|);
+           push @$cmd, $mountpoint;
+           push @$cmd, '--conf', $configfile if defined($configfile);
+    }else {
+       my $source = "$server:$subdir";
+       $cmd = ['/bin/mount', '-t', 'ceph', $source, $mountpoint, '-o', 
"name=$username"];
+       push @$cmd, '-o', "secretfile=$keyfile" if (-e $keyfile);
+    }
+
+    if ($scfg->{options}) {
+       push @$cmd, '-o', $scfg->{options};
+    }
+
+    run_command($cmd, errmsg => "mount error");
+}
+
+# Configuration
+
+sub type {
+    return 'cephfs';
+}
+
+sub plugindata {
+    return {
+       content => [ { vztmpl => 1, iso => 1, backup => 1},
+                    { backup => 1 }],
+       format => [ { raw => 1, qcow2 => 1, vmdk => 1 } , 'raw' ],
+    };
+}
+
+sub properties {
+    return {
+       fuse => {
+           description => "Mount cephfs through fuse.",
+           type => 'boolean',
+       },
+       subdir => {
+           description => "Subdir to mount.",
+           type => 'string', format => 'pve-storage-path',
+       },
+    };
+}
+
+sub options {
+    return {
+       path => { fixed => 1 },
+       monhost => { optional => 1},
+       nodes => { optional => 1 },
+       subdir => { optional => 1 },
+       disable => { optional => 1 },
+       options => { optional => 1 },
+       username => { optional => 1 },
+       content => { optional => 1 },
+       format => { optional => 1 },
+       mkdir => { optional => 1 },
+       fuse => { optional => 1 },
+       bwlimit => { optional => 1 },
+    };
+}
+
+sub check_config {
+    my ($class, $sectionId, $config, $create, $skipSchemaCheck) = @_;
+
+    $config->{path} = "/mnt/pve/$sectionId" if $create && !$config->{path};
+
+    return $class->SUPER::check_config($sectionId, $config, $create, 
$skipSchemaCheck);
+}
+
+# Storage implementation
+
+sub status {
+    my ($class, $storeid, $scfg, $cache) = @_;
+
+    $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts()
+       if !$cache->{mountdata};
+
+    return undef if !cephfs_is_mounted($scfg, $storeid, $cache->{mountdata});
+
+    return $class->SUPER::status($storeid, $scfg, $cache);
+}
+
+sub activate_storage {
+    my ($class, $storeid, $scfg, $cache) = @_;
+
+    $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts()
+       if !$cache->{mountdata};
+
+    my $path = $scfg->{path};
+
+    if (!cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) {
+
+       # NOTE: only call mkpath when not mounted (avoid hang
+       # when cephfs is offline
+
+       mkpath $path if !(defined($scfg->{mkdir}) && !$scfg->{mkdir});
+
+       die "unable to activate storage '$storeid' - " .
+           "directory '$path' does not exist\n" if ! -d $path;
+
+       cephfs_mount($scfg, $storeid);
+    }
+
+    $class->SUPER::activate_storage($storeid, $scfg, $cache);
+}
+
+sub deactivate_storage {
+    my ($class, $storeid, $scfg, $cache) = @_;
+
+    $cache->{mountdata} = PVE::ProcFSTools::parse_proc_mounts()
+       if !$cache->{mountdata};
+
+    my $path = $scfg->{path};
+
+    if (cephfs_is_mounted($scfg, $storeid, $cache->{mountdata})) {
+       my $cmd = ['/bin/umount', $path];
+       run_command($cmd, errmsg => 'umount error'); 
+    }
+}
diff --git a/PVE/Storage/Makefile b/PVE/Storage/Makefile
index 7b168fa..ad69532 100644
--- a/PVE/Storage/Makefile
+++ b/PVE/Storage/Makefile
@@ -1,4 +1,4 @@
-SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm 
ISCSIPlugin.pm RBDPlugin.pm SheepdogPlugin.pm ISCSIDirectPlugin.pm 
GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm DRBDPlugin.pm LvmThinPlugin.pm
+SOURCES=Plugin.pm DirPlugin.pm LVMPlugin.pm NFSPlugin.pm CIFSPlugin.pm 
ISCSIPlugin.pm CephFSPlugin.pm RBDPlugin.pm SheepdogPlugin.pm 
ISCSIDirectPlugin.pm GlusterfsPlugin.pm ZFSPoolPlugin.pm ZFSPlugin.pm 
DRBDPlugin.pm LvmThinPlugin.pm
 
 .PHONY: install
 install:
diff --git a/PVE/Storage/Plugin.pm b/PVE/Storage/Plugin.pm
index 163871d..3769e01 100644
--- a/PVE/Storage/Plugin.pm
+++ b/PVE/Storage/Plugin.pm
@@ -24,6 +24,7 @@ our @SHARED_STORAGE = (
     'nfs',
     'cifs',
     'rbd',
+    'cephfs',
     'sheepdog',
     'iscsidirect',
     'glusterfs',
diff --git a/debian/control b/debian/control
index 2cf585a..70b5e2d 100644
--- a/debian/control
+++ b/debian/control
@@ -28,6 +28,8 @@ Depends: cstream,
          udev,
          smbclient,
          cifs-utils,
+         ceph-common,
+         ceph-fuse,
          ${perl:Depends},
 Description: Proxmox VE storage management library
  This package contains the storage management library used by Proxmox VE.
-- 
2.11.0


_______________________________________________
pve-devel mailing list
pve-devel@pve.proxmox.com
https://pve.proxmox.com/cgi-bin/mailman/listinfo/pve-devel

Reply via email to