The following pull request was submitted through Github.
It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/6136

This e-mail was sent by the LXC bot, direct replies will not reach the author
unless they happen to be subscribed to this list.

=== Description (from pull-request) ===

From 8b173cae21615d224c960b82741a8615f3bff581 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.h...@canonical.com>
Date: Mon, 19 Aug 2019 19:36:04 +0200
Subject: [PATCH 1/6] lxd: Move Create{Container,Snapshot}Mountpoint to storage

Signed-off-by: Thomas Hipp <thomas.h...@canonical.com>
---
 lxd/api_internal.go           |  4 +--
 lxd/container_post.go         |  4 +--
 lxd/patches.go                | 10 +++---
 lxd/storage.go                | 55 --------------------------------
 lxd/storage/storage.go        | 60 +++++++++++++++++++++++++++++++++++
 lxd/storage_btrfs.go          | 16 +++++-----
 lxd/storage_ceph.go           |  6 ++--
 lxd/storage_ceph_migration.go |  2 +-
 lxd/storage_ceph_utils.go     |  8 ++---
 lxd/storage_dir.go            | 14 ++++----
 lxd/storage_lvm.go            | 10 +++---
 lxd/storage_lvm_utils.go      |  8 ++---
 lxd/storage_zfs.go            | 14 ++++----
 lxd/storage_zfs_utils.go      |  4 +--
 14 files changed, 110 insertions(+), 105 deletions(-)

diff --git a/lxd/api_internal.go b/lxd/api_internal.go
index ddfffe5ae8..cbe27ad22e 100644
--- a/lxd/api_internal.go
+++ b/lxd/api_internal.go
@@ -923,7 +923,7 @@ func internalImport(d *Daemon, r *http.Request) Response {
        if backup.Container.Config["security.privileged"] == "" {
                isPrivileged = true
        }
-       err = createContainerMountpoint(containerMntPoint, containerPath,
+       err = driver.CreateContainerMountpoint(containerMntPoint, containerPath,
                isPrivileged)
        if err != nil {
                return InternalError(err)
@@ -1029,7 +1029,7 @@ func internalImport(d *Daemon, r *http.Request) Response {
                sourceName = project.Prefix(projectName, sourceName)
                snapshotMntPointSymlinkTarget := 
shared.VarPath("storage-pools", backup.Pool.Name, "containers-snapshots", 
sourceName)
                snapshotMntPointSymlink := shared.VarPath("snapshots", 
sourceName)
-               err = createSnapshotMountpoint(snapshotMountPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+               err = driver.CreateSnapshotMountpoint(snapshotMountPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
                if err != nil {
                        return InternalError(err)
                }
diff --git a/lxd/container_post.go b/lxd/container_post.go
index 5883dc69c9..8bd33b2dfa 100644
--- a/lxd/container_post.go
+++ b/lxd/container_post.go
@@ -531,7 +531,7 @@ func containerPostCreateContainerMountPoint(d *Daemon, 
project, containerName st
        }
 
        containerMntPoint := driver.GetContainerMountPoint(c.Project(), 
poolName, containerName)
-       err = createContainerMountpoint(containerMntPoint, c.Path(), 
c.IsPrivileged())
+       err = driver.CreateContainerMountpoint(containerMntPoint, c.Path(), 
c.IsPrivileged())
        if err != nil {
                return errors.Wrap(err, "Failed to create container mount point 
on target node")
        }
@@ -541,7 +541,7 @@ func containerPostCreateContainerMountPoint(d *Daemon, 
project, containerName st
                snapshotsSymlinkTarget := shared.VarPath("storage-pools",
                        poolName, "containers-snapshots", containerName)
                snapshotMntPointSymlink := shared.VarPath("snapshots", 
containerName)
-               err := createSnapshotMountpoint(mntPoint, 
snapshotsSymlinkTarget, snapshotMntPointSymlink)
+               err := driver.CreateSnapshotMountpoint(mntPoint, 
snapshotsSymlinkTarget, snapshotMntPointSymlink)
                if err != nil {
                        return errors.Wrap(err, "Failed to create snapshot 
mount point on target node")
                }
diff --git a/lxd/patches.go b/lxd/patches.go
index 0ee421fa12..4ba86582f7 100644
--- a/lxd/patches.go
+++ b/lxd/patches.go
@@ -524,7 +524,7 @@ func upgradeFromStorageTypeBtrfs(name string, d *Daemon, 
defaultPoolName string,
                // ${LXD_DIR}/containers/<container_name> to
                // ${LXD_DIR}/storage-pools/<pool>/containers/<container_name>
                doesntMatter := false
-               err = createContainerMountpoint(newContainerMntPoint, 
oldContainerMntPoint, doesntMatter)
+               err = driver.CreateContainerMountpoint(newContainerMntPoint, 
oldContainerMntPoint, doesntMatter)
                if err != nil {
                        return err
                }
@@ -808,7 +808,7 @@ func upgradeFromStorageTypeDir(name string, d *Daemon, 
defaultPoolName string, d
                }
 
                doesntMatter := false
-               err = createContainerMountpoint(newContainerMntPoint, 
oldContainerMntPoint, doesntMatter)
+               err = driver.CreateContainerMountpoint(newContainerMntPoint, 
oldContainerMntPoint, doesntMatter)
                if err != nil {
                        return err
                }
@@ -855,7 +855,7 @@ func upgradeFromStorageTypeDir(name string, d *Daemon, 
defaultPoolName string, d
                }
 
                // Create a symlink for this container.  snapshots.
-               err = createSnapshotMountpoint(newSnapshotMntPoint, 
newSnapshotMntPoint, oldSnapshotMntPoint)
+               err = driver.CreateSnapshotMountpoint(newSnapshotMntPoint, 
newSnapshotMntPoint, oldSnapshotMntPoint)
                if err != nil {
                        return err
                }
@@ -1191,7 +1191,7 @@ func upgradeFromStorageTypeLvm(name string, d *Daemon, 
defaultPoolName string, d
 
                // Create the new container mountpoint.
                doesntMatter := false
-               err = createContainerMountpoint(newContainerMntPoint, 
oldContainerMntPoint, doesntMatter)
+               err = driver.CreateContainerMountpoint(newContainerMntPoint, 
oldContainerMntPoint, doesntMatter)
                if err != nil {
                        logger.Errorf("Failed to create container mountpoint 
\"%s\" for LVM logical volume: %s", newContainerMntPoint, err)
                        return err
@@ -1632,7 +1632,7 @@ func upgradeFromStorageTypeZfs(name string, d *Daemon, 
defaultPoolName string, d
                // the path but in case it somehow didn't let's do it ourselves.
                doesntMatter := false
                newContainerMntPoint := 
driver.GetContainerMountPoint("default", poolName, ct)
-               err = createContainerMountpoint(newContainerMntPoint, 
oldContainerMntPoint, doesntMatter)
+               err = driver.CreateContainerMountpoint(newContainerMntPoint, 
oldContainerMntPoint, doesntMatter)
                if err != nil {
                        logger.Warnf("Failed to create mountpoint for the 
container: %s", newContainerMntPoint)
                        failedUpgradeEntities = append(failedUpgradeEntities, 
fmt.Sprintf("containers/%s: Failed to create container mountpoint: %s", ct, 
err))
diff --git a/lxd/storage.go b/lxd/storage.go
index 5ee427f4e8..1a03e2fb56 100644
--- a/lxd/storage.go
+++ b/lxd/storage.go
@@ -607,40 +607,6 @@ func storagePoolVolumeContainerLoadInit(s *state.State, 
project, containerName s
        return storagePoolVolumeInit(s, project, poolName, containerName, 
storagePoolVolumeTypeContainer)
 }
 
-func createContainerMountpoint(mountPoint string, mountPointSymlink string, 
privileged bool) error {
-       var mode os.FileMode
-       if privileged {
-               mode = 0700
-       } else {
-               mode = 0711
-       }
-
-       mntPointSymlinkExist := shared.PathExists(mountPointSymlink)
-       mntPointSymlinkTargetExist := shared.PathExists(mountPoint)
-
-       var err error
-       if !mntPointSymlinkTargetExist {
-               err = os.MkdirAll(mountPoint, 0711)
-               if err != nil {
-                       return err
-               }
-       }
-
-       err = os.Chmod(mountPoint, mode)
-       if err != nil {
-               return err
-       }
-
-       if !mntPointSymlinkExist {
-               err := os.Symlink(mountPoint, mountPointSymlink)
-               if err != nil {
-                       return err
-               }
-       }
-
-       return nil
-}
-
 func deleteContainerMountpoint(mountPoint string, mountPointSymlink string, 
storageTypeName string) error {
        if shared.PathExists(mountPointSymlink) {
                err := os.Remove(mountPointSymlink)
@@ -698,27 +664,6 @@ func renameContainerMountpoint(oldMountPoint string, 
oldMountPointSymlink string
        return nil
 }
 
-func createSnapshotMountpoint(snapshotMountpoint string, 
snapshotsSymlinkTarget string, snapshotsSymlink string) error {
-       snapshotMntPointExists := shared.PathExists(snapshotMountpoint)
-       mntPointSymlinkExist := shared.PathExists(snapshotsSymlink)
-
-       if !snapshotMntPointExists {
-               err := os.MkdirAll(snapshotMountpoint, 0711)
-               if err != nil {
-                       return err
-               }
-       }
-
-       if !mntPointSymlinkExist {
-               err := os.Symlink(snapshotsSymlinkTarget, snapshotsSymlink)
-               if err != nil {
-                       return err
-               }
-       }
-
-       return nil
-}
-
 func deleteSnapshotMountpoint(snapshotMountpoint string, 
snapshotsSymlinkTarget string, snapshotsSymlink string) error {
        if shared.PathExists(snapshotMountpoint) {
                err := os.Remove(snapshotMountpoint)
diff --git a/lxd/storage/storage.go b/lxd/storage/storage.go
index f55c9f4ef4..b8d8d42b31 100644
--- a/lxd/storage/storage.go
+++ b/lxd/storage/storage.go
@@ -1,6 +1,8 @@
 package storage
 
 import (
+       "os"
+
        "github.com/lxc/lxd/lxd/project"
        "github.com/lxc/lxd/shared"
 )
@@ -49,3 +51,61 @@ func GetStoragePoolVolumeMountPoint(poolName string, 
volumeName string) string {
 func GetStoragePoolVolumeSnapshotMountPoint(poolName string, snapshotName 
string) string {
        return shared.VarPath("storage-pools", poolName, "custom-snapshots", 
snapshotName)
 }
+
+// CreateContainerMountpoint creates the provided container mountpoint and 
symlink.
+func CreateContainerMountpoint(mountPoint string, mountPointSymlink string, 
privileged bool) error {
+       var mode os.FileMode
+       if privileged {
+               mode = 0700
+       } else {
+               mode = 0711
+       }
+
+       mntPointSymlinkExist := shared.PathExists(mountPointSymlink)
+       mntPointSymlinkTargetExist := shared.PathExists(mountPoint)
+
+       var err error
+       if !mntPointSymlinkTargetExist {
+               err = os.MkdirAll(mountPoint, 0711)
+               if err != nil {
+                       return err
+               }
+       }
+
+       err = os.Chmod(mountPoint, mode)
+       if err != nil {
+               return err
+       }
+
+       if !mntPointSymlinkExist {
+               err := os.Symlink(mountPoint, mountPointSymlink)
+               if err != nil {
+                       return err
+               }
+       }
+
+       return nil
+}
+
+// CreateSnapshotMountpoint creates the provided container snapshot mountpoint
+// and symlink.
+func CreateSnapshotMountpoint(snapshotMountpoint string, 
snapshotsSymlinkTarget string, snapshotsSymlink string) error {
+       snapshotMntPointExists := shared.PathExists(snapshotMountpoint)
+       mntPointSymlinkExist := shared.PathExists(snapshotsSymlink)
+
+       if !snapshotMntPointExists {
+               err := os.MkdirAll(snapshotMountpoint, 0711)
+               if err != nil {
+                       return err
+               }
+       }
+
+       if !mntPointSymlinkExist {
+               err := os.Symlink(snapshotsSymlinkTarget, snapshotsSymlink)
+               if err != nil {
+                       return err
+               }
+       }
+
+       return nil
+}
diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go
index b9e133dd16..392fa71726 100644
--- a/lxd/storage_btrfs.go
+++ b/lxd/storage_btrfs.go
@@ -836,7 +836,7 @@ func (s *storageBtrfs) doContainerCreate(projectName, name 
string, privileged bo
 
        // Create the mountpoint for the container at:
        // ${LXD_DIR}/containers/<name>
-       err = createContainerMountpoint(containerSubvolumeName, 
shared.VarPath("containers", project.Prefix(projectName, name)), privileged)
+       err = driver.CreateContainerMountpoint(containerSubvolumeName, 
shared.VarPath("containers", project.Prefix(projectName, name)), privileged)
        if err != nil {
                return err
        }
@@ -925,7 +925,7 @@ func (s *storageBtrfs) ContainerCreateFromImage(container 
container, fingerprint
 
        // Create the mountpoint for the container at:
        // ${LXD_DIR}/containers/<name>
-       err = createContainerMountpoint(containerSubvolumeName, 
container.Path(), container.IsPrivileged())
+       err = driver.CreateContainerMountpoint(containerSubvolumeName, 
container.Path(), container.IsPrivileged())
        if err != nil {
                return errors.Wrap(err, "Failed to create container mountpoint")
        }
@@ -1006,7 +1006,7 @@ func (s *storageBtrfs) copyContainer(target container, 
source container) error {
                return err
        }
 
-       err = createContainerMountpoint(targetContainerSubvolumeName, 
target.Path(), target.IsPrivileged())
+       err = driver.CreateContainerMountpoint(targetContainerSubvolumeName, 
target.Path(), target.IsPrivileged())
        if err != nil {
                return err
        }
@@ -1029,7 +1029,7 @@ func (s *storageBtrfs) copySnapshot(target container, 
source container) error {
        containersPath := driver.GetSnapshotMountPoint(target.Project(), 
s.pool.Name, targetParentName)
        snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", 
s.pool.Name, "containers-snapshots", project.Prefix(target.Project(), 
targetParentName))
        snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(target.Project(), targetParentName))
-       err := createSnapshotMountpoint(containersPath, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+       err := driver.CreateSnapshotMountpoint(containersPath, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
        if err != nil {
                return err
        }
@@ -1581,7 +1581,7 @@ func (s *storageBtrfs) 
ContainerSnapshotCreateEmpty(snapshotContainer container)
        snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", 
s.pool.Name, "containers-snapshots", 
project.Prefix(snapshotContainer.Project(), sourceName))
        snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(snapshotContainer.Project(), sourceName))
        if !shared.PathExists(snapshotMntPointSymlink) {
-               err := createContainerMountpoint(snapshotMntPointSymlinkTarget, 
snapshotMntPointSymlink, snapshotContainer.IsPrivileged())
+               err := 
driver.CreateContainerMountpoint(snapshotMntPointSymlinkTarget, 
snapshotMntPointSymlink, snapshotContainer.IsPrivileged())
                if err != nil {
                        return err
                }
@@ -1857,7 +1857,7 @@ func (s *storageBtrfs) 
doContainerBackupLoadOptimized(info backupInfo, data io.R
                snapshotMntPoint := driver.GetSnapshotMountPoint(info.Project, 
s.pool.Name, containerName)
                snapshotMntPointSymlinkTarget := 
shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", 
project.Prefix(info.Project, containerName))
                snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(info.Project, containerName))
-               err = createSnapshotMountpoint(snapshotMntPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+               err = driver.CreateSnapshotMountpoint(snapshotMntPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
                if err != nil {
                        feeder.Close()
                        return err
@@ -1900,7 +1900,7 @@ func (s *storageBtrfs) 
doContainerBackupLoadOptimized(info backupInfo, data io.R
        }
 
        // Create mountpoints
-       err = createContainerMountpoint(containerMntPoint, 
shared.VarPath("containers", project.Prefix(info.Project, info.Name)), 
info.Privileged)
+       err = driver.CreateContainerMountpoint(containerMntPoint, 
shared.VarPath("containers", project.Prefix(info.Project, info.Name)), 
info.Privileged)
        if err != nil {
                return err
        }
@@ -2777,7 +2777,7 @@ func (s *storageBtrfs) MigrationSink(conn 
*websocket.Conn, op *operation, args M
 
                        snapshotMntPointSymlinkTarget := 
shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", 
project.Prefix(args.Container.Project(), containerName))
                        snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(args.Container.Project(), containerName))
-                       err = createSnapshotMountpoint(snapshotMntPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+                       err = driver.CreateSnapshotMountpoint(snapshotMntPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
                        if err != nil {
                                return err
                        }
diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go
index bd7bbbd4e8..09b7ffc367 100644
--- a/lxd/storage_ceph.go
+++ b/lxd/storage_ceph.go
@@ -878,7 +878,7 @@ func (s *storageCeph) ContainerCreateFromImage(container 
container, fingerprint
 
        // Create the mountpoint
        privileged := container.IsPrivileged()
-       err = createContainerMountpoint(containerPoolVolumeMntPoint,
+       err = driver.CreateContainerMountpoint(containerPoolVolumeMntPoint,
                containerPath, privileged)
        if err != nil {
                logger.Errorf(`Failed to create mountpoint "%s" for container 
"%s" for RBD storage volume: %s`, containerPoolVolumeMntPoint, containerName, 
err)
@@ -1135,7 +1135,7 @@ func (s *storageCeph) ContainerCopy(target container, 
source container,
                        target.Project(),
                        s.pool.Name,
                        targetContainerName)
-               err = createContainerMountpoint(
+               err = driver.CreateContainerMountpoint(
                        targetContainerMountPoint,
                        targetContainerPath,
                        target.IsPrivileged())
@@ -1248,7 +1248,7 @@ func (s *storageCeph) ContainerCopy(target container, 
source container,
                                "snapshots",
                                project.Prefix(target.Project(), 
targetContainerName))
 
-                       err := createSnapshotMountpoint(
+                       err := driver.CreateSnapshotMountpoint(
                                containersPath,
                                snapshotMntPointSymlinkTarget,
                                snapshotMntPointSymlink)
diff --git a/lxd/storage_ceph_migration.go b/lxd/storage_ceph_migration.go
index 70e37a3e82..57e7839ef1 100644
--- a/lxd/storage_ceph_migration.go
+++ b/lxd/storage_ceph_migration.go
@@ -352,7 +352,7 @@ func (s *storageCeph) MigrationSink(conn *websocket.Conn, 
op *operation, args Mi
        }
 
        containerMntPoint := 
driver.GetContainerMountPoint(args.Container.Project(), s.pool.Name, 
containerName)
-       err = createContainerMountpoint(
+       err = driver.CreateContainerMountpoint(
                containerMntPoint,
                args.Container.Path(),
                args.Container.IsPrivileged())
diff --git a/lxd/storage_ceph_utils.go b/lxd/storage_ceph_utils.go
index 104b151f4c..1a82a510f6 100644
--- a/lxd/storage_ceph_utils.go
+++ b/lxd/storage_ceph_utils.go
@@ -768,7 +768,7 @@ func (s *storageCeph) copyWithoutSnapshotsFull(target 
container,
 
        // Create mountpoint
        targetContainerMountPoint := 
driver.GetContainerMountPoint(target.Project(), s.pool.Name, target.Name())
-       err = createContainerMountpoint(targetContainerMountPoint, 
target.Path(), target.IsPrivileged())
+       err = driver.CreateContainerMountpoint(targetContainerMountPoint, 
target.Path(), target.IsPrivileged())
        if err != nil {
                return err
        }
@@ -849,7 +849,7 @@ func (s *storageCeph) copyWithoutSnapshotsSparse(target 
container,
 
        // Create mountpoint
        targetContainerMountPoint := 
driver.GetContainerMountPoint(target.Project(), s.pool.Name, target.Name())
-       err = createContainerMountpoint(targetContainerMountPoint, 
target.Path(), target.IsPrivileged())
+       err = driver.CreateContainerMountpoint(targetContainerMountPoint, 
target.Path(), target.IsPrivileged())
        if err != nil {
                return err
        }
@@ -1763,7 +1763,7 @@ func (s *storageCeph) doContainerCreate(projectName, name 
string, privileged boo
 
        containerPath := shared.VarPath("containers", 
project.Prefix(projectName, name))
        containerMntPoint := driver.GetContainerMountPoint(projectName, 
s.pool.Name, name)
-       err = createContainerMountpoint(containerMntPoint, containerPath, 
privileged)
+       err = driver.CreateContainerMountpoint(containerMntPoint, 
containerPath, privileged)
        if err != nil {
                logger.Errorf(`Failed to create mountpoint "%s" for RBD storage 
volume for container "%s" on storage pool "%s": %s"`, containerMntPoint, name, 
s.pool.Name, err)
                return err
@@ -1876,7 +1876,7 @@ func (s *storageCeph) 
doContainerSnapshotCreate(projectName, targetName string,
        sourceOnlyName, _, _ := 
shared.ContainerGetParentAndSnapshotName(sourceName)
        snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", 
s.pool.Name, "containers-snapshots", project.Prefix(projectName, 
sourceOnlyName))
        snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(projectName, sourceOnlyName))
-       err = createSnapshotMountpoint(targetContainerMntPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+       err = driver.CreateSnapshotMountpoint(targetContainerMntPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
        if err != nil {
                logger.Errorf(`Failed to create mountpoint "%s", snapshot 
symlink target "%s", snapshot mountpoint symlink"%s" for RBD storage volume 
"%s" on storage pool "%s": %s`, targetContainerMntPoint, 
snapshotMntPointSymlinkTarget,
                        snapshotMntPointSymlink, s.volume.Name, s.pool.Name, 
err)
diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go
index 21646c6111..3c38cedcfd 100644
--- a/lxd/storage_dir.go
+++ b/lxd/storage_dir.go
@@ -508,7 +508,7 @@ func (s *storageDir) ContainerCreate(container container) 
error {
        }
 
        containerMntPoint := driver.GetContainerMountPoint(container.Project(), 
s.pool.Name, container.Name())
-       err = createContainerMountpoint(containerMntPoint, container.Path(), 
container.IsPrivileged())
+       err = driver.CreateContainerMountpoint(containerMntPoint, 
container.Path(), container.IsPrivileged())
        if err != nil {
                return err
        }
@@ -552,7 +552,7 @@ func (s *storageDir) ContainerCreateFromImage(container 
container, imageFingerpr
        privileged := container.IsPrivileged()
        containerName := container.Name()
        containerMntPoint := driver.GetContainerMountPoint(container.Project(), 
s.pool.Name, containerName)
-       err = createContainerMountpoint(containerMntPoint, container.Path(), 
privileged)
+       err = driver.CreateContainerMountpoint(containerMntPoint, 
container.Path(), privileged)
        if err != nil {
                return errors.Wrap(err, "Create container mount point")
        }
@@ -657,7 +657,7 @@ func (s *storageDir) copyContainer(target container, source 
container) error {
        }
        targetContainerMntPoint := 
driver.GetContainerMountPoint(target.Project(), targetPool, target.Name())
 
-       err := createContainerMountpoint(targetContainerMntPoint, 
target.Path(), target.IsPrivileged())
+       err := driver.CreateContainerMountpoint(targetContainerMntPoint, 
target.Path(), target.IsPrivileged())
        if err != nil {
                return err
        }
@@ -691,7 +691,7 @@ func (s *storageDir) copySnapshot(target container, 
targetPool string, source co
        containersPath := driver.GetSnapshotMountPoint(target.Project(), 
targetPool, targetParentName)
        snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", 
targetPool, "containers-snapshots", project.Prefix(target.Project(), 
targetParentName))
        snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(target.Project(), targetParentName))
-       err := createSnapshotMountpoint(containersPath, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+       err := driver.CreateSnapshotMountpoint(containersPath, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
        if err != nil {
                return err
        }
@@ -1027,7 +1027,7 @@ func (s *storageDir) 
ContainerSnapshotCreateEmpty(snapshotContainer container) e
        snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools",
                s.pool.Name, "containers-snapshots", 
project.Prefix(snapshotContainer.Project(), sourceName))
        snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(snapshotContainer.Project(), sourceName))
-       err = createSnapshotMountpoint(targetContainerMntPoint,
+       err = driver.CreateSnapshotMountpoint(targetContainerMntPoint,
                snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
        if err != nil {
                return err
@@ -1222,7 +1222,7 @@ func (s *storageDir) ContainerBackupLoad(info backupInfo, 
data io.ReadSeeker, ta
 
        // Create mountpoints
        containerMntPoint := driver.GetContainerMountPoint(info.Project, 
s.pool.Name, info.Name)
-       err = createContainerMountpoint(containerMntPoint, 
driver.ContainerPath(project.Prefix(info.Project, info.Name), false), 
info.Privileged)
+       err = driver.CreateContainerMountpoint(containerMntPoint, 
driver.ContainerPath(project.Prefix(info.Project, info.Name), false), 
info.Privileged)
        if err != nil {
                return errors.Wrap(err, "Create container mount point")
        }
@@ -1248,7 +1248,7 @@ func (s *storageDir) ContainerBackupLoad(info backupInfo, 
data io.ReadSeeker, ta
                snapshotMntPointSymlinkTarget := 
shared.VarPath("storage-pools", s.pool.Name,
                        "containers-snapshots", project.Prefix(info.Project, 
info.Name))
                snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(info.Project, info.Name))
-               err := createSnapshotMountpoint(snapshotMntPoint, 
snapshotMntPointSymlinkTarget,
+               err := driver.CreateSnapshotMountpoint(snapshotMntPoint, 
snapshotMntPointSymlinkTarget,
                        snapshotMntPointSymlink)
                if err != nil {
                        return err
diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go
index 1a52f99e54..7f2b1f619d 100644
--- a/lxd/storage_lvm.go
+++ b/lxd/storage_lvm.go
@@ -965,7 +965,7 @@ func (s *storageLvm) ContainerCreate(container container) 
error {
                if err != nil {
                        return err
                }
-               err = createSnapshotMountpoint(containerMntPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+               err = driver.CreateSnapshotMountpoint(containerMntPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
                if err != nil {
                        return err
                }
@@ -976,7 +976,7 @@ func (s *storageLvm) ContainerCreate(container container) 
error {
                if err != nil {
                        return err
                }
-               err = createContainerMountpoint(containerMntPoint, 
containerPath, container.IsPrivileged())
+               err = driver.CreateContainerMountpoint(containerMntPoint, 
containerPath, container.IsPrivileged())
                if err != nil {
                        return err
                }
@@ -1019,7 +1019,7 @@ func (s *storageLvm) ContainerCreateFromImage(container 
container, fingerprint s
        if err != nil {
                return errors.Wrapf(err, "Create container mount point 
directory at %s", containerMntPoint)
        }
-       err = createContainerMountpoint(containerMntPoint, containerPath, 
container.IsPrivileged())
+       err = driver.CreateContainerMountpoint(containerMntPoint, 
containerPath, container.IsPrivileged())
        if err != nil {
                return errors.Wrap(err, "Create container mount point")
        }
@@ -1878,10 +1878,10 @@ func (s *storageLvm) doContainerBackupLoad(projectName, 
containerName string, pr
                cname, _, _ := 
shared.ContainerGetParentAndSnapshotName(containerName)
                snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(projectName, cname))
                snapshotMntPointSymlinkTarget := 
shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", 
project.Prefix(projectName, cname))
-               err = createSnapshotMountpoint(containerMntPoint, 
snapshotMntPointSymlinkTarget,
+               err = driver.CreateSnapshotMountpoint(containerMntPoint, 
snapshotMntPointSymlinkTarget,
                        snapshotMntPointSymlink)
        } else {
-               err = createContainerMountpoint(containerMntPoint, 
containerPath, privileged)
+               err = driver.CreateContainerMountpoint(containerMntPoint, 
containerPath, privileged)
        }
        if err != nil {
                return "", err
diff --git a/lxd/storage_lvm_utils.go b/lxd/storage_lvm_utils.go
index ff92174139..3c88a36888 100644
--- a/lxd/storage_lvm_utils.go
+++ b/lxd/storage_lvm_utils.go
@@ -288,10 +288,10 @@ func (s *storageLvm) 
createSnapshotContainer(snapshotContainer container, source
                sourceName, _, _ := 
shared.ContainerGetParentAndSnapshotName(sourceContainerName)
                snapshotMntPointSymlinkTarget := 
shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", 
project.Prefix(sourceContainer.Project(), sourceName))
                snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(sourceContainer.Project(), sourceName))
-               err = createSnapshotMountpoint(targetContainerMntPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+               err = driver.CreateSnapshotMountpoint(targetContainerMntPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
        } else {
                targetContainerMntPoint = 
driver.GetContainerMountPoint(sourceContainer.Project(), targetPool, 
targetContainerName)
-               err = createContainerMountpoint(targetContainerMntPoint, 
targetContainerPath, snapshotContainer.IsPrivileged())
+               err = driver.CreateContainerMountpoint(targetContainerMntPoint, 
targetContainerPath, snapshotContainer.IsPrivileged())
        }
        if err != nil {
                return errors.Wrap(err, "Create mount point")
@@ -351,7 +351,7 @@ func (s *storageLvm) copySnapshot(target container, source 
container, refresh bo
        containersPath := driver.GetSnapshotMountPoint(target.Project(), 
s.pool.Name, targetParentName)
        snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", 
s.pool.Name, "containers-snapshots", project.Prefix(target.Project(), 
targetParentName))
        snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(target.Project(), targetParentName))
-       err = createSnapshotMountpoint(containersPath, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+       err = driver.CreateSnapshotMountpoint(containersPath, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
        if err != nil {
                return err
        }
@@ -452,7 +452,7 @@ func (s *storageLvm) copyContainer(target container, source 
container, refresh b
        }
 
        targetContainerMntPoint := 
driver.GetContainerMountPoint(target.Project(), targetPool, target.Name())
-       err = createContainerMountpoint(targetContainerMntPoint, target.Path(), 
target.IsPrivileged())
+       err = driver.CreateContainerMountpoint(targetContainerMntPoint, 
target.Path(), target.IsPrivileged())
        if err != nil {
                return err
        }
diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go
index 97b2901923..d467cfc364 100644
--- a/lxd/storage_zfs.go
+++ b/lxd/storage_zfs.go
@@ -900,7 +900,7 @@ func (s *storageZfs) ContainerCreateFromImage(container 
container, fingerprint s
        }
 
        privileged := container.IsPrivileged()
-       err = createContainerMountpoint(containerPoolVolumeMntPoint, 
containerPath, privileged)
+       err = driver.CreateContainerMountpoint(containerPoolVolumeMntPoint, 
containerPath, privileged)
        if err != nil {
                return err
        }
@@ -988,7 +988,7 @@ func (s *storageZfs) copyWithoutSnapshotsSparse(target 
container, source contain
                        defer s.ContainerUmount(target, targetContainerPath)
                }
 
-               err = createContainerMountpoint(targetContainerMountPoint, 
targetContainerPath, target.IsPrivileged())
+               err = 
driver.CreateContainerMountpoint(targetContainerMountPoint, 
targetContainerPath, target.IsPrivileged())
                if err != nil {
                        return err
                }
@@ -1119,7 +1119,7 @@ func (s *storageZfs) copyWithoutSnapshotFull(target 
container, source container)
                defer s.ContainerUmount(target, targetContainerMountPoint)
        }
 
-       err = createContainerMountpoint(targetContainerMountPoint, 
target.Path(), target.IsPrivileged())
+       err = driver.CreateContainerMountpoint(targetContainerMountPoint, 
target.Path(), target.IsPrivileged())
        if err != nil {
                return err
        }
@@ -1134,7 +1134,7 @@ func (s *storageZfs) copyWithSnapshots(target container, 
source container, paren
        containersPath := driver.GetSnapshotMountPoint(target.Project(), 
s.pool.Name, targetParentName)
        snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", 
s.pool.Name, "containers-snapshots", project.Prefix(target.Project(), 
targetParentName))
        snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(target.Project(), targetParentName))
-       err := createSnapshotMountpoint(containersPath, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+       err := driver.CreateSnapshotMountpoint(containersPath, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
        if err != nil {
                return err
        }
@@ -1291,7 +1291,7 @@ func (s *storageZfs) ContainerCopy(target container, 
source container, container
                targetContainerName := target.Name()
                targetContainerPath := target.Path()
                targetContainerMountPoint := 
driver.GetContainerMountPoint(target.Project(), s.pool.Name, 
targetContainerName)
-               err = createContainerMountpoint(targetContainerMountPoint, 
targetContainerPath, target.IsPrivileged())
+               err = 
driver.CreateContainerMountpoint(targetContainerMountPoint, 
targetContainerPath, target.IsPrivileged())
                if err != nil {
                        return err
                }
@@ -2132,7 +2132,7 @@ func (s *storageZfs) ContainerBackupCreate(backup backup, 
source container) erro
 func (s *storageZfs) doContainerBackupLoadOptimized(info backupInfo, data 
io.ReadSeeker, tarArgs []string) error {
        containerName, _, _ := 
shared.ContainerGetParentAndSnapshotName(info.Name)
        containerMntPoint := driver.GetContainerMountPoint(info.Project, 
s.pool.Name, containerName)
-       err := createContainerMountpoint(containerMntPoint, 
driver.ContainerPath(info.Name, false), info.Privileged)
+       err := driver.CreateContainerMountpoint(containerMntPoint, 
driver.ContainerPath(info.Name, false), info.Privileged)
        if err != nil {
                return err
        }
@@ -2192,7 +2192,7 @@ func (s *storageZfs) doContainerBackupLoadOptimized(info 
backupInfo, data io.Rea
                snapshotMntPoint := driver.GetSnapshotMountPoint(info.Project, 
s.pool.Name, fmt.Sprintf("%s/%s", containerName, snapshotOnlyName))
                snapshotMntPointSymlinkTarget := 
shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", 
project.Prefix(info.Project, containerName))
                snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(info.Project, containerName))
-               err = createSnapshotMountpoint(snapshotMntPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
+               err = driver.CreateSnapshotMountpoint(snapshotMntPoint, 
snapshotMntPointSymlinkTarget, snapshotMntPointSymlink)
                if err != nil {
                        // can't use defer because it needs to run before the 
mount
                        os.RemoveAll(unpackPath)
diff --git a/lxd/storage_zfs_utils.go b/lxd/storage_zfs_utils.go
index 5944c40407..4a2d87091f 100644
--- a/lxd/storage_zfs_utils.go
+++ b/lxd/storage_zfs_utils.go
@@ -670,7 +670,7 @@ func (s *storageZfs) doContainerMount(projectName, name 
string, privileged bool)
        // Since we're using mount() directly zfs will not automatically create
        // the mountpoint for us. So let's check and do it if needed.
        if !shared.PathExists(containerPoolVolumeMntPoint) {
-               err := createContainerMountpoint(containerPoolVolumeMntPoint, 
shared.VarPath(fs), privileged)
+               err := 
driver.CreateContainerMountpoint(containerPoolVolumeMntPoint, 
shared.VarPath(fs), privileged)
                if err != nil {
                        return false, err
                }
@@ -810,7 +810,7 @@ func (s *storageZfs) doContainerCreate(projectName, name 
string, privileged bool
                return err
        }
 
-       err = createContainerMountpoint(containerPoolVolumeMntPoint, 
containerPath, privileged)
+       err = driver.CreateContainerMountpoint(containerPoolVolumeMntPoint, 
containerPath, privileged)
        if err != nil {
                return err
        }

From e81185b3a8c511152d68ede3e3adf1533454cf70 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.h...@canonical.com>
Date: Mon, 19 Aug 2019 19:37:26 +0200
Subject: [PATCH 2/6] lxd: Move ChangeableStoragePoolProperties to storage

Signed-off-by: Thomas Hipp <thomas.h...@canonical.com>
---
 lxd/storage/pools_config.go | 36 ++++++++++++++++++++++++++++++++++++
 lxd/storage_btrfs.go        |  2 +-
 lxd/storage_ceph.go         |  4 ++--
 lxd/storage_cephfs.go       |  2 +-
 lxd/storage_dir.go          |  2 +-
 lxd/storage_lvm.go          |  2 +-
 lxd/storage_pools_config.go | 30 ------------------------------
 lxd/storage_zfs.go          |  2 +-
 8 files changed, 43 insertions(+), 37 deletions(-)
 create mode 100644 lxd/storage/pools_config.go

diff --git a/lxd/storage/pools_config.go b/lxd/storage/pools_config.go
new file mode 100644
index 0000000000..6583acb168
--- /dev/null
+++ b/lxd/storage/pools_config.go
@@ -0,0 +1,36 @@
+package storage
+
+import "fmt"
+
+// ChangeableStoragePoolProperties is a map of storage backends and their
+// changeable storage pool properties.
+var ChangeableStoragePoolProperties = map[string][]string{
+       "btrfs": {
+               "rsync.bwlimit",
+               "btrfs.mount_options"},
+
+       "ceph": {
+               "volume.block.filesystem",
+               "volume.block.mount_options",
+               "volume.size"},
+
+       "cephfs": {
+               "rsync.bwlimit"},
+
+       "dir": {
+               "rsync.bwlimit"},
+
+       "lvm": {
+               "lvm.thinpool_name",
+               "lvm.vg_name",
+               "volume.block.filesystem",
+               "volume.block.mount_options",
+               "volume.size"},
+
+       "zfs": {
+               "rsync_bwlimit",
+               "volume.zfs.remove_snapshots",
+               "volume.zfs.use_refquota",
+               "zfs.clone_copy"},
+}
+
diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go
index 392fa71726..c5221f95b2 100644
--- a/lxd/storage_btrfs.go
+++ b/lxd/storage_btrfs.go
@@ -511,7 +511,7 @@ func (s *storageBtrfs) StoragePoolUpdate(writable 
*api.StoragePoolPut,
        changedConfig []string) error {
        logger.Infof(`Updating BTRFS storage pool "%s"`, s.pool.Name)
 
-       changeable := changeableStoragePoolProperties["btrfs"]
+       changeable := driver.ChangeableStoragePoolProperties["btrfs"]
        unchangeable := []string{}
        for _, change := range changedConfig {
                if !shared.StringInSlice(change, changeable) {
diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go
index 09b7ffc367..ddbc1d7f16 100644
--- a/lxd/storage_ceph.go
+++ b/lxd/storage_ceph.go
@@ -735,7 +735,7 @@ func (s *storageCeph) StoragePoolVolumeRename(newName 
string) error {
 func (s *storageCeph) StoragePoolUpdate(writable *api.StoragePoolPut, 
changedConfig []string) error {
        logger.Infof(`Updating CEPH storage pool "%s"`, s.pool.Name)
 
-       changeable := changeableStoragePoolProperties["ceph"]
+       changeable := driver.ChangeableStoragePoolProperties["ceph"]
        unchangeable := []string{}
        for _, change := range changedConfig {
                if !shared.StringInSlice(change, changeable) {
@@ -744,7 +744,7 @@ func (s *storageCeph) StoragePoolUpdate(writable 
*api.StoragePoolPut, changedCon
        }
 
        if len(unchangeable) > 0 {
-               return updateStoragePoolError(unchangeable, "ceph")
+               return driver.UpdateStoragePoolError(unchangeable, "ceph")
        }
 
        // "rsync.bwlimit" requires no on-disk modifications.
diff --git a/lxd/storage_cephfs.go b/lxd/storage_cephfs.go
index 305de37498..b0457fd8cf 100644
--- a/lxd/storage_cephfs.go
+++ b/lxd/storage_cephfs.go
@@ -429,7 +429,7 @@ func (s *storageCephFs) StoragePoolUpdate(writable 
*api.StoragePoolPut, changedC
        logger.Infof(`Updating CEPHFS storage pool "%s"`, s.pool.Name)
 
        // Validate the properties
-       changeable := changeableStoragePoolProperties["cephfs"]
+       changeable := driver.ChangeableStoragePoolProperties["cephfs"]
        unchangeable := []string{}
        for _, change := range changedConfig {
                if !shared.StringInSlice(change, changeable) {
diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go
index 3c38cedcfd..e9f437aa1c 100644
--- a/lxd/storage_dir.go
+++ b/lxd/storage_dir.go
@@ -291,7 +291,7 @@ func (s *storageDir) StoragePoolUpdate(writable 
*api.StoragePoolPut, changedConf
                return err
        }
 
-       changeable := changeableStoragePoolProperties["dir"]
+       changeable := driver.ChangeableStoragePoolProperties["dir"]
        unchangeable := []string{}
        for _, change := range changedConfig {
                if !shared.StringInSlice(change, changeable) {
diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go
index 7f2b1f619d..ff97f8027f 100644
--- a/lxd/storage_lvm.go
+++ b/lxd/storage_lvm.go
@@ -688,7 +688,7 @@ func (s *storageLvm) GetContainerPoolInfo() (int64, string, 
string) {
 func (s *storageLvm) StoragePoolUpdate(writable *api.StoragePoolPut, 
changedConfig []string) error {
        logger.Infof(`Updating LVM storage pool "%s"`, s.pool.Name)
 
-       changeable := changeableStoragePoolProperties["lvm"]
+       changeable := driver.ChangeableStoragePoolProperties["lvm"]
        unchangeable := []string{}
        for _, change := range changedConfig {
                if !shared.StringInSlice(change, changeable) {
diff --git a/lxd/storage_pools_config.go b/lxd/storage_pools_config.go
index b44f0659df..1f06c0deee 100644
--- a/lxd/storage_pools_config.go
+++ b/lxd/storage_pools_config.go
@@ -16,36 +16,6 @@ func updateStoragePoolError(unchangeable []string, 
driverName string) error {
                `storage pools`, unchangeable, driverName)
 }
 
-var changeableStoragePoolProperties = map[string][]string{
-       "btrfs": {
-               "rsync.bwlimit",
-               "btrfs.mount_options"},
-
-       "ceph": {
-               "volume.block.filesystem",
-               "volume.block.mount_options",
-               "volume.size"},
-
-       "cephfs": {
-               "rsync.bwlimit"},
-
-       "dir": {
-               "rsync.bwlimit"},
-
-       "lvm": {
-               "lvm.thinpool_name",
-               "lvm.vg_name",
-               "volume.block.filesystem",
-               "volume.block.mount_options",
-               "volume.size"},
-
-       "zfs": {
-               "rsync_bwlimit",
-               "volume.zfs.remove_snapshots",
-               "volume.zfs.use_refquota",
-               "zfs.clone_copy"},
-}
-
 var storagePoolConfigKeys = map[string]func(value string) error{
        // valid drivers: btrfs
        // (Note, that we can't be smart in detecting mount options since a lot
diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go
index d467cfc364..0d98cf2977 100644
--- a/lxd/storage_zfs.go
+++ b/lxd/storage_zfs.go
@@ -629,7 +629,7 @@ func (s *storageZfs) GetContainerPoolInfo() (int64, string, 
string) {
 func (s *storageZfs) StoragePoolUpdate(writable *api.StoragePoolPut, 
changedConfig []string) error {
        logger.Infof(`Updating ZFS storage pool "%s"`, s.pool.Name)
 
-       changeable := changeableStoragePoolProperties["zfs"]
+       changeable := driver.ChangeableStoragePoolProperties["zfs"]
        unchangeable := []string{}
        for _, change := range changedConfig {
                if !shared.StringInSlice(change, changeable) {

From cd1f65f6807644d87236bf47b09c7a09724c0c69 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.h...@canonical.com>
Date: Thu, 29 Aug 2019 20:38:29 +0200
Subject: [PATCH 3/6] lxd: Move UpdateStoragePoolError to storage

Signed-off-by: Thomas Hipp <thomas.h...@canonical.com>
---
 lxd/storage/pools_config.go | 6 ++++++
 lxd/storage_btrfs.go        | 2 +-
 lxd/storage_cephfs.go       | 2 +-
 lxd/storage_dir.go          | 2 +-
 lxd/storage_lvm.go          | 2 +-
 lxd/storage_pools_config.go | 5 -----
 lxd/storage_zfs.go          | 2 +-
 7 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/lxd/storage/pools_config.go b/lxd/storage/pools_config.go
index 6583acb168..a0c1684c53 100644
--- a/lxd/storage/pools_config.go
+++ b/lxd/storage/pools_config.go
@@ -34,3 +34,9 @@ var ChangeableStoragePoolProperties = map[string][]string{
                "zfs.clone_copy"},
 }
 
+// UpdateStoragePoolError returns an error stating that the provided properties
+// cannot be changed in the given backend.
+func UpdateStoragePoolError(unchangeable []string, driverName string) error {
+       return fmt.Errorf("The %v properties cannot be changed for %q storage 
pools",
+               unchangeable, driverName)
+}
diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go
index c5221f95b2..3c10da58fd 100644
--- a/lxd/storage_btrfs.go
+++ b/lxd/storage_btrfs.go
@@ -520,7 +520,7 @@ func (s *storageBtrfs) StoragePoolUpdate(writable 
*api.StoragePoolPut,
        }
 
        if len(unchangeable) > 0 {
-               return updateStoragePoolError(unchangeable, "btrfs")
+               return driver.UpdateStoragePoolError(unchangeable, "btrfs")
        }
 
        // "rsync.bwlimit" requires no on-disk modifications.
diff --git a/lxd/storage_cephfs.go b/lxd/storage_cephfs.go
index b0457fd8cf..3fe207b143 100644
--- a/lxd/storage_cephfs.go
+++ b/lxd/storage_cephfs.go
@@ -438,7 +438,7 @@ func (s *storageCephFs) StoragePoolUpdate(writable 
*api.StoragePoolPut, changedC
        }
 
        if len(unchangeable) > 0 {
-               return updateStoragePoolError(unchangeable, "cephfs")
+               return driver.UpdateStoragePoolError(unchangeable, "cephfs")
        }
 
        logger.Infof(`Updated CEPHFS storage pool "%s"`, s.pool.Name)
diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go
index e9f437aa1c..736700304b 100644
--- a/lxd/storage_dir.go
+++ b/lxd/storage_dir.go
@@ -300,7 +300,7 @@ func (s *storageDir) StoragePoolUpdate(writable 
*api.StoragePoolPut, changedConf
        }
 
        if len(unchangeable) > 0 {
-               return updateStoragePoolError(unchangeable, "dir")
+               return driver.UpdateStoragePoolError(unchangeable, "dir")
        }
 
        // "rsync.bwlimit" requires no on-disk modifications.
diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go
index ff97f8027f..ff8e06b703 100644
--- a/lxd/storage_lvm.go
+++ b/lxd/storage_lvm.go
@@ -697,7 +697,7 @@ func (s *storageLvm) StoragePoolUpdate(writable 
*api.StoragePoolPut, changedConf
        }
 
        if len(unchangeable) > 0 {
-               return updateStoragePoolError(unchangeable, "lvm")
+               return driver.UpdateStoragePoolError(unchangeable, "lvm")
        }
 
        // "volume.block.mount_options" requires no on-disk modifications.
diff --git a/lxd/storage_pools_config.go b/lxd/storage_pools_config.go
index 1f06c0deee..6ecd411ade 100644
--- a/lxd/storage_pools_config.go
+++ b/lxd/storage_pools_config.go
@@ -11,11 +11,6 @@ import (
        "github.com/lxc/lxd/shared/units"
 )
 
-func updateStoragePoolError(unchangeable []string, driverName string) error {
-       return fmt.Errorf(`The %v properties cannot be changed for "%s" `+
-               `storage pools`, unchangeable, driverName)
-}
-
 var storagePoolConfigKeys = map[string]func(value string) error{
        // valid drivers: btrfs
        // (Note, that we can't be smart in detecting mount options since a lot
diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go
index 0d98cf2977..0087bd098c 100644
--- a/lxd/storage_zfs.go
+++ b/lxd/storage_zfs.go
@@ -638,7 +638,7 @@ func (s *storageZfs) StoragePoolUpdate(writable 
*api.StoragePoolPut, changedConf
        }
 
        if len(unchangeable) > 0 {
-               return updateStoragePoolError(unchangeable, "zfs")
+               return driver.UpdateStoragePoolError(unchangeable, "zfs")
        }
 
        // "rsync.bwlimit" requires no on-disk modifications.

From b3b3bad101ee03bcc15f4768df5a54eb28d3c887 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.h...@canonical.com>
Date: Fri, 30 Aug 2019 19:51:46 +0200
Subject: [PATCH 4/6] lxd: Move btrfs migration code

The migration code needs to be moved to its own file since
storage_btrfs.go will be removed in the future. The code however needs
to stay in the main package (for now) since it depends on the container
interface.

Signed-off-by: Thomas Hipp <thomas.h...@canonical.com>
---
 lxd/storage_btrfs.go           | 170 ------------------------------
 lxd/storage_migration_btrfs.go | 185 +++++++++++++++++++++++++++++++++
 2 files changed, 185 insertions(+), 170 deletions(-)
 create mode 100644 lxd/storage_migration_btrfs.go

diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go
index 3c10da58fd..f1cf4cd842 100644
--- a/lxd/storage_btrfs.go
+++ b/lxd/storage_btrfs.go
@@ -2430,170 +2430,6 @@ func btrfsSubVolumesGet(path string) ([]string, error) {
        return result, nil
 }
 
-type btrfsMigrationSourceDriver struct {
-       container          container
-       snapshots          []container
-       btrfsSnapshotNames []string
-       btrfs              *storageBtrfs
-       runningSnapName    string
-       stoppedSnapName    string
-}
-
-func (s *btrfsMigrationSourceDriver) send(conn *websocket.Conn, btrfsPath 
string, btrfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) 
error {
-       args := []string{"send"}
-       if btrfsParent != "" {
-               args = append(args, "-p", btrfsParent)
-       }
-       args = append(args, btrfsPath)
-
-       cmd := exec.Command("btrfs", args...)
-
-       stdout, err := cmd.StdoutPipe()
-       if err != nil {
-               return err
-       }
-
-       readPipe := io.ReadCloser(stdout)
-       if readWrapper != nil {
-               readPipe = readWrapper(stdout)
-       }
-
-       stderr, err := cmd.StderrPipe()
-       if err != nil {
-               return err
-       }
-
-       err = cmd.Start()
-       if err != nil {
-               return err
-       }
-
-       <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024)
-
-       output, err := ioutil.ReadAll(stderr)
-       if err != nil {
-               logger.Errorf("Problem reading btrfs send stderr: %s", err)
-       }
-
-       err = cmd.Wait()
-       if err != nil {
-               logger.Errorf("Problem with btrfs send: %s", string(output))
-       }
-
-       return err
-}
-
-func (s *btrfsMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn, op 
*operation, bwlimit string, containerOnly bool) error {
-       _, containerPool, _ := s.container.Storage().GetContainerPoolInfo()
-       containerName := s.container.Name()
-       containersPath := driver.GetContainerMountPoint("default", 
containerPool, "")
-       sourceName := containerName
-
-       // Deal with sending a snapshot to create a container on another LXD
-       // instance.
-       if s.container.IsSnapshot() {
-               sourceName, _, _ := 
shared.ContainerGetParentAndSnapshotName(containerName)
-               snapshotsPath := 
driver.GetSnapshotMountPoint(s.container.Project(), containerPool, sourceName)
-               tmpContainerMntPoint, err := ioutil.TempDir(snapshotsPath, 
sourceName)
-               if err != nil {
-                       return err
-               }
-               defer os.RemoveAll(tmpContainerMntPoint)
-
-               err = os.Chmod(tmpContainerMntPoint, 0700)
-               if err != nil {
-                       return err
-               }
-
-               migrationSendSnapshot := fmt.Sprintf("%s/.migration-send", 
tmpContainerMntPoint)
-               snapshotMntPoint := 
driver.GetSnapshotMountPoint(s.container.Project(), containerPool, 
containerName)
-               err = s.btrfs.btrfsPoolVolumesSnapshot(snapshotMntPoint, 
migrationSendSnapshot, true, true)
-               if err != nil {
-                       return err
-               }
-               defer btrfsSubVolumesDelete(migrationSendSnapshot)
-
-               wrapper := StorageProgressReader(op, "fs_progress", 
containerName)
-               return s.send(conn, migrationSendSnapshot, "", wrapper)
-       }
-
-       if !containerOnly {
-               for i, snap := range s.snapshots {
-                       prev := ""
-                       if i > 0 {
-                               prev = 
driver.GetSnapshotMountPoint(snap.Project(), containerPool, 
s.snapshots[i-1].Name())
-                       }
-
-                       snapMntPoint := 
driver.GetSnapshotMountPoint(snap.Project(), containerPool, snap.Name())
-                       wrapper := StorageProgressReader(op, "fs_progress", 
snap.Name())
-                       if err := s.send(conn, snapMntPoint, prev, wrapper); 
err != nil {
-                               return err
-                       }
-               }
-       }
-
-       tmpContainerMntPoint, err := ioutil.TempDir(containersPath, 
containerName)
-       if err != nil {
-               return err
-       }
-       defer os.RemoveAll(tmpContainerMntPoint)
-
-       err = os.Chmod(tmpContainerMntPoint, 0700)
-       if err != nil {
-               return err
-       }
-
-       migrationSendSnapshot := fmt.Sprintf("%s/.migration-send", 
tmpContainerMntPoint)
-       containerMntPoint := 
driver.GetContainerMountPoint(s.container.Project(), containerPool, sourceName)
-       err = s.btrfs.btrfsPoolVolumesSnapshot(containerMntPoint, 
migrationSendSnapshot, true, true)
-       if err != nil {
-               return err
-       }
-       defer btrfsSubVolumesDelete(migrationSendSnapshot)
-
-       btrfsParent := ""
-       if len(s.btrfsSnapshotNames) > 0 {
-               btrfsParent = s.btrfsSnapshotNames[len(s.btrfsSnapshotNames)-1]
-       }
-
-       wrapper := StorageProgressReader(op, "fs_progress", containerName)
-       return s.send(conn, migrationSendSnapshot, btrfsParent, wrapper)
-}
-
-func (s *btrfsMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, 
bwlimit string) error {
-       tmpPath := driver.GetSnapshotMountPoint(s.container.Project(), 
s.btrfs.pool.Name,
-               fmt.Sprintf("%s/.migration-send", s.container.Name()))
-       err := os.MkdirAll(tmpPath, 0711)
-       if err != nil {
-               return err
-       }
-
-       err = os.Chmod(tmpPath, 0700)
-       if err != nil {
-               return err
-       }
-
-       s.stoppedSnapName = fmt.Sprintf("%s/.root", tmpPath)
-       parentName, _, _ := 
shared.ContainerGetParentAndSnapshotName(s.container.Name())
-       containerMntPt := driver.GetContainerMountPoint(s.container.Project(), 
s.btrfs.pool.Name, parentName)
-       err = s.btrfs.btrfsPoolVolumesSnapshot(containerMntPt, 
s.stoppedSnapName, true, true)
-       if err != nil {
-               return err
-       }
-
-       return s.send(conn, s.stoppedSnapName, s.runningSnapName, nil)
-}
-
-func (s *btrfsMigrationSourceDriver) Cleanup() {
-       if s.stoppedSnapName != "" {
-               btrfsSubVolumesDelete(s.stoppedSnapName)
-       }
-
-       if s.runningSnapName != "" {
-               btrfsSubVolumesDelete(s.runningSnapName)
-       }
-}
-
 func (s *storageBtrfs) MigrationType() migration.MigrationFSType {
        if s.s.OS.RunningInUserNS {
                return migration.MigrationFSType_RSYNC
@@ -3121,12 +2957,6 @@ func (s *storageBtrfs) doCrossPoolVolumeCopy(sourcePool 
string, sourceName strin
        return nil
 }
 
-func (s *btrfsMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, 
op *operation, bwlimit string, storage storage, volumeOnly bool) error {
-       msg := fmt.Sprintf("Function not implemented")
-       logger.Errorf(msg)
-       return fmt.Errorf(msg)
-}
-
 func (s *storageBtrfs) StorageMigrationSource(args MigrationSourceArgs) 
(MigrationStorageSourceDriver, error) {
        return rsyncStorageMigrationSource(args)
 }
diff --git a/lxd/storage_migration_btrfs.go b/lxd/storage_migration_btrfs.go
new file mode 100644
index 0000000000..2f58187467
--- /dev/null
+++ b/lxd/storage_migration_btrfs.go
@@ -0,0 +1,185 @@
+package main
+
+import (
+       "fmt"
+       "io"
+       "io/ioutil"
+       "os"
+       "os/exec"
+
+       "github.com/gorilla/websocket"
+
+       driver "github.com/lxc/lxd/lxd/storage"
+       "github.com/lxc/lxd/shared"
+       "github.com/lxc/lxd/shared/logger"
+)
+
+type btrfsMigrationSourceDriver struct {
+       container          container
+       snapshots          []container
+       btrfsSnapshotNames []string
+       btrfs              *storageBtrfs
+       runningSnapName    string
+       stoppedSnapName    string
+}
+
+func (s *btrfsMigrationSourceDriver) send(conn *websocket.Conn, btrfsPath 
string, btrfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) 
error {
+       args := []string{"send"}
+       if btrfsParent != "" {
+               args = append(args, "-p", btrfsParent)
+       }
+       args = append(args, btrfsPath)
+
+       cmd := exec.Command("btrfs", args...)
+
+       stdout, err := cmd.StdoutPipe()
+       if err != nil {
+               return err
+       }
+
+       readPipe := io.ReadCloser(stdout)
+       if readWrapper != nil {
+               readPipe = readWrapper(stdout)
+       }
+
+       stderr, err := cmd.StderrPipe()
+       if err != nil {
+               return err
+       }
+
+       err = cmd.Start()
+       if err != nil {
+               return err
+       }
+
+       <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024)
+
+       output, err := ioutil.ReadAll(stderr)
+       if err != nil {
+               logger.Errorf("Problem reading btrfs send stderr: %s", err)
+       }
+
+       err = cmd.Wait()
+       if err != nil {
+               logger.Errorf("Problem with btrfs send: %s", string(output))
+       }
+
+       return err
+}
+
+func (s *btrfsMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn, op 
*operation, bwlimit string, containerOnly bool) error {
+       _, containerPool, _ := s.container.Storage().GetContainerPoolInfo()
+       containerName := s.container.Name()
+       containersPath := driver.GetContainerMountPoint("default", 
containerPool, "")
+       sourceName := containerName
+
+       // Deal with sending a snapshot to create a container on another LXD
+       // instance.
+       if s.container.IsSnapshot() {
+               sourceName, _, _ := 
shared.ContainerGetParentAndSnapshotName(containerName)
+               snapshotsPath := 
driver.GetSnapshotMountPoint(s.container.Project(), containerPool, sourceName)
+               tmpContainerMntPoint, err := ioutil.TempDir(snapshotsPath, 
sourceName)
+               if err != nil {
+                       return err
+               }
+               defer os.RemoveAll(tmpContainerMntPoint)
+
+               err = os.Chmod(tmpContainerMntPoint, 0700)
+               if err != nil {
+                       return err
+               }
+
+               migrationSendSnapshot := fmt.Sprintf("%s/.migration-send", 
tmpContainerMntPoint)
+               snapshotMntPoint := 
driver.GetSnapshotMountPoint(s.container.Project(), containerPool, 
containerName)
+               err = s.btrfs.btrfsPoolVolumesSnapshot(snapshotMntPoint, 
migrationSendSnapshot, true, true)
+               if err != nil {
+                       return err
+               }
+               defer btrfsSubVolumesDelete(migrationSendSnapshot)
+
+               wrapper := StorageProgressReader(op, "fs_progress", 
containerName)
+               return s.send(conn, migrationSendSnapshot, "", wrapper)
+       }
+
+       if !containerOnly {
+               for i, snap := range s.snapshots {
+                       prev := ""
+                       if i > 0 {
+                               prev = 
driver.GetSnapshotMountPoint(snap.Project(), containerPool, 
s.snapshots[i-1].Name())
+                       }
+
+                       snapMntPoint := 
driver.GetSnapshotMountPoint(snap.Project(), containerPool, snap.Name())
+                       wrapper := StorageProgressReader(op, "fs_progress", 
snap.Name())
+                       if err := s.send(conn, snapMntPoint, prev, wrapper); 
err != nil {
+                               return err
+                       }
+               }
+       }
+
+       tmpContainerMntPoint, err := ioutil.TempDir(containersPath, 
containerName)
+       if err != nil {
+               return err
+       }
+       defer os.RemoveAll(tmpContainerMntPoint)
+
+       err = os.Chmod(tmpContainerMntPoint, 0700)
+       if err != nil {
+               return err
+       }
+
+       migrationSendSnapshot := fmt.Sprintf("%s/.migration-send", 
tmpContainerMntPoint)
+       containerMntPoint := 
driver.GetContainerMountPoint(s.container.Project(), containerPool, sourceName)
+       err = s.btrfs.btrfsPoolVolumesSnapshot(containerMntPoint, 
migrationSendSnapshot, true, true)
+       if err != nil {
+               return err
+       }
+       defer btrfsSubVolumesDelete(migrationSendSnapshot)
+
+       btrfsParent := ""
+       if len(s.btrfsSnapshotNames) > 0 {
+               btrfsParent = s.btrfsSnapshotNames[len(s.btrfsSnapshotNames)-1]
+       }
+
+       wrapper := StorageProgressReader(op, "fs_progress", containerName)
+       return s.send(conn, migrationSendSnapshot, btrfsParent, wrapper)
+}
+
+func (s *btrfsMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, 
bwlimit string) error {
+       tmpPath := driver.GetSnapshotMountPoint(s.container.Project(), 
s.btrfs.pool.Name,
+               fmt.Sprintf("%s/.migration-send", s.container.Name()))
+       err := os.MkdirAll(tmpPath, 0711)
+       if err != nil {
+               return err
+       }
+
+       err = os.Chmod(tmpPath, 0700)
+       if err != nil {
+               return err
+       }
+
+       s.stoppedSnapName = fmt.Sprintf("%s/.root", tmpPath)
+       parentName, _, _ := 
shared.ContainerGetParentAndSnapshotName(s.container.Name())
+       containerMntPt := driver.GetContainerMountPoint(s.container.Project(), 
s.btrfs.pool.Name, parentName)
+       err = s.btrfs.btrfsPoolVolumesSnapshot(containerMntPt, 
s.stoppedSnapName, true, true)
+       if err != nil {
+               return err
+       }
+
+       return s.send(conn, s.stoppedSnapName, s.runningSnapName, nil)
+}
+
+func (s *btrfsMigrationSourceDriver) Cleanup() {
+       if s.stoppedSnapName != "" {
+               btrfsSubVolumesDelete(s.stoppedSnapName)
+       }
+
+       if s.runningSnapName != "" {
+               btrfsSubVolumesDelete(s.runningSnapName)
+       }
+}
+
+func (s *btrfsMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, 
op *operation, bwlimit string, storage storage, volumeOnly bool) error {
+       msg := fmt.Sprintf("Function not implemented")
+       logger.Errorf(msg)
+       return fmt.Errorf(msg)
+}

From 2264dbe92e20196f7d1713184a3e18083dd843dc Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.h...@canonical.com>
Date: Fri, 30 Aug 2019 20:04:04 +0200
Subject: [PATCH 5/6] lxd: Move zfs migration code

The migration code needs to be moved to its own file since
storage_zfs.go will be removed in the future. The code however needs
to stay in the main package (for now) since it depends on the container
interface.

Signed-off-by: Thomas Hipp <thomas.h...@canonical.com>
---
 lxd/storage_migration_zfs.go | 146 +++++++++++++++++++++++++++++++++++
 lxd/storage_zfs.go           | 131 -------------------------------
 2 files changed, 146 insertions(+), 131 deletions(-)
 create mode 100644 lxd/storage_migration_zfs.go

diff --git a/lxd/storage_migration_zfs.go b/lxd/storage_migration_zfs.go
new file mode 100644
index 0000000000..fcfad12e5c
--- /dev/null
+++ b/lxd/storage_migration_zfs.go
@@ -0,0 +1,146 @@
+package main
+
+import (
+       "fmt"
+       "io"
+       "io/ioutil"
+       "os/exec"
+
+       "github.com/gorilla/websocket"
+       "github.com/pborman/uuid"
+
+       "github.com/lxc/lxd/lxd/project"
+       "github.com/lxc/lxd/shared"
+       "github.com/lxc/lxd/shared/logger"
+)
+
+type zfsMigrationSourceDriver struct {
+       container        container
+       snapshots        []container
+       zfsSnapshotNames []string
+       zfs              *storageZfs
+       runningSnapName  string
+       stoppedSnapName  string
+       zfsFeatures      []string
+}
+
+func (s *zfsMigrationSourceDriver) send(conn *websocket.Conn, zfsName string, 
zfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) error {
+       sourceParentName, _, _ := 
shared.ContainerGetParentAndSnapshotName(s.container.Name())
+       poolName := s.zfs.getOnDiskPoolName()
+       args := []string{"send"}
+
+       // Negotiated options
+       if s.zfsFeatures != nil && len(s.zfsFeatures) > 0 {
+               if shared.StringInSlice("compress", s.zfsFeatures) {
+                       args = append(args, "-c")
+                       args = append(args, "-L")
+               }
+       }
+
+       args = append(args, []string{fmt.Sprintf("%s/containers/%s@%s", 
poolName, project.Prefix(s.container.Project(), sourceParentName), zfsName)}...)
+       if zfsParent != "" {
+               args = append(args, "-i", fmt.Sprintf("%s/containers/%s@%s", 
poolName, project.Prefix(s.container.Project(), s.container.Name()), zfsParent))
+       }
+
+       cmd := exec.Command("zfs", args...)
+
+       stdout, err := cmd.StdoutPipe()
+       if err != nil {
+               return err
+       }
+
+       readPipe := io.ReadCloser(stdout)
+       if readWrapper != nil {
+               readPipe = readWrapper(stdout)
+       }
+
+       stderr, err := cmd.StderrPipe()
+       if err != nil {
+               return err
+       }
+
+       if err := cmd.Start(); err != nil {
+               return err
+       }
+
+       <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024)
+
+       output, err := ioutil.ReadAll(stderr)
+       if err != nil {
+               logger.Errorf("Problem reading zfs send stderr: %s", err)
+       }
+
+       err = cmd.Wait()
+       if err != nil {
+               logger.Errorf("Problem with zfs send: %s", string(output))
+       }
+
+       return err
+}
+
+func (s *zfsMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn, op 
*operation, bwlimit string, containerOnly bool) error {
+       if s.container.IsSnapshot() {
+               _, snapOnlyName, _ := 
shared.ContainerGetParentAndSnapshotName(s.container.Name())
+               snapshotName := fmt.Sprintf("snapshot-%s", snapOnlyName)
+               wrapper := StorageProgressReader(op, "fs_progress", 
s.container.Name())
+               return s.send(conn, snapshotName, "", wrapper)
+       }
+
+       lastSnap := ""
+       if !containerOnly {
+               for i, snap := range s.zfsSnapshotNames {
+                       prev := ""
+                       if i > 0 {
+                               prev = s.zfsSnapshotNames[i-1]
+                       }
+
+                       lastSnap = snap
+
+                       wrapper := StorageProgressReader(op, "fs_progress", 
snap)
+                       if err := s.send(conn, snap, prev, wrapper); err != nil 
{
+                               return err
+                       }
+               }
+       }
+
+       s.runningSnapName = fmt.Sprintf("migration-send-%s", 
uuid.NewRandom().String())
+       if err := zfsPoolVolumeSnapshotCreate(s.zfs.getOnDiskPoolName(), 
fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), 
s.container.Name())), s.runningSnapName); err != nil {
+               return err
+       }
+
+       wrapper := StorageProgressReader(op, "fs_progress", s.container.Name())
+       if err := s.send(conn, s.runningSnapName, lastSnap, wrapper); err != 
nil {
+               return err
+       }
+
+       return nil
+}
+
+func (s *zfsMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, 
bwlimit string) error {
+       s.stoppedSnapName = fmt.Sprintf("migration-send-%s", 
uuid.NewRandom().String())
+       if err := zfsPoolVolumeSnapshotCreate(s.zfs.getOnDiskPoolName(), 
fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), 
s.container.Name())), s.stoppedSnapName); err != nil {
+               return err
+       }
+
+       if err := s.send(conn, s.stoppedSnapName, s.runningSnapName, nil); err 
!= nil {
+               return err
+       }
+
+       return nil
+}
+
+func (s *zfsMigrationSourceDriver) Cleanup() {
+       poolName := s.zfs.getOnDiskPoolName()
+       if s.stoppedSnapName != "" {
+               zfsPoolVolumeSnapshotDestroy(poolName, 
fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), 
s.container.Name())), s.stoppedSnapName)
+       }
+       if s.runningSnapName != "" {
+               zfsPoolVolumeSnapshotDestroy(poolName, 
fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), 
s.container.Name())), s.runningSnapName)
+       }
+}
+
+func (s *zfsMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, op 
*operation, bwlimit string, storage storage, volumeOnly bool) error {
+       msg := fmt.Sprintf("Function not implemented")
+       logger.Errorf(msg)
+       return fmt.Errorf(msg)
+}
diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go
index 0087bd098c..93585d1ba6 100644
--- a/lxd/storage_zfs.go
+++ b/lxd/storage_zfs.go
@@ -2506,131 +2506,6 @@ func (s *storageZfs) ImageUmount(fingerprint string) 
(bool, error) {
        return true, nil
 }
 
-type zfsMigrationSourceDriver struct {
-       container        container
-       snapshots        []container
-       zfsSnapshotNames []string
-       zfs              *storageZfs
-       runningSnapName  string
-       stoppedSnapName  string
-       zfsFeatures      []string
-}
-
-func (s *zfsMigrationSourceDriver) send(conn *websocket.Conn, zfsName string, 
zfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) error {
-       sourceParentName, _, _ := 
shared.ContainerGetParentAndSnapshotName(s.container.Name())
-       poolName := s.zfs.getOnDiskPoolName()
-       args := []string{"send"}
-
-       // Negotiated options
-       if s.zfsFeatures != nil && len(s.zfsFeatures) > 0 {
-               if shared.StringInSlice("compress", s.zfsFeatures) {
-                       args = append(args, "-c")
-                       args = append(args, "-L")
-               }
-       }
-
-       args = append(args, []string{fmt.Sprintf("%s/containers/%s@%s", 
poolName, project.Prefix(s.container.Project(), sourceParentName), zfsName)}...)
-       if zfsParent != "" {
-               args = append(args, "-i", fmt.Sprintf("%s/containers/%s@%s", 
poolName, project.Prefix(s.container.Project(), s.container.Name()), zfsParent))
-       }
-
-       cmd := exec.Command("zfs", args...)
-
-       stdout, err := cmd.StdoutPipe()
-       if err != nil {
-               return err
-       }
-
-       readPipe := io.ReadCloser(stdout)
-       if readWrapper != nil {
-               readPipe = readWrapper(stdout)
-       }
-
-       stderr, err := cmd.StderrPipe()
-       if err != nil {
-               return err
-       }
-
-       if err := cmd.Start(); err != nil {
-               return err
-       }
-
-       <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024)
-
-       output, err := ioutil.ReadAll(stderr)
-       if err != nil {
-               logger.Errorf("Problem reading zfs send stderr: %s", err)
-       }
-
-       err = cmd.Wait()
-       if err != nil {
-               logger.Errorf("Problem with zfs send: %s", string(output))
-       }
-
-       return err
-}
-
-func (s *zfsMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn, op 
*operation, bwlimit string, containerOnly bool) error {
-       if s.container.IsSnapshot() {
-               _, snapOnlyName, _ := 
shared.ContainerGetParentAndSnapshotName(s.container.Name())
-               snapshotName := fmt.Sprintf("snapshot-%s", snapOnlyName)
-               wrapper := StorageProgressReader(op, "fs_progress", 
s.container.Name())
-               return s.send(conn, snapshotName, "", wrapper)
-       }
-
-       lastSnap := ""
-       if !containerOnly {
-               for i, snap := range s.zfsSnapshotNames {
-                       prev := ""
-                       if i > 0 {
-                               prev = s.zfsSnapshotNames[i-1]
-                       }
-
-                       lastSnap = snap
-
-                       wrapper := StorageProgressReader(op, "fs_progress", 
snap)
-                       if err := s.send(conn, snap, prev, wrapper); err != nil 
{
-                               return err
-                       }
-               }
-       }
-
-       s.runningSnapName = fmt.Sprintf("migration-send-%s", 
uuid.NewRandom().String())
-       if err := zfsPoolVolumeSnapshotCreate(s.zfs.getOnDiskPoolName(), 
fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), 
s.container.Name())), s.runningSnapName); err != nil {
-               return err
-       }
-
-       wrapper := StorageProgressReader(op, "fs_progress", s.container.Name())
-       if err := s.send(conn, s.runningSnapName, lastSnap, wrapper); err != 
nil {
-               return err
-       }
-
-       return nil
-}
-
-func (s *zfsMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, 
bwlimit string) error {
-       s.stoppedSnapName = fmt.Sprintf("migration-send-%s", 
uuid.NewRandom().String())
-       if err := zfsPoolVolumeSnapshotCreate(s.zfs.getOnDiskPoolName(), 
fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), 
s.container.Name())), s.stoppedSnapName); err != nil {
-               return err
-       }
-
-       if err := s.send(conn, s.stoppedSnapName, s.runningSnapName, nil); err 
!= nil {
-               return err
-       }
-
-       return nil
-}
-
-func (s *zfsMigrationSourceDriver) Cleanup() {
-       poolName := s.zfs.getOnDiskPoolName()
-       if s.stoppedSnapName != "" {
-               zfsPoolVolumeSnapshotDestroy(poolName, 
fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), 
s.container.Name())), s.stoppedSnapName)
-       }
-       if s.runningSnapName != "" {
-               zfsPoolVolumeSnapshotDestroy(poolName, 
fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), 
s.container.Name())), s.runningSnapName)
-       }
-}
-
 func (s *storageZfs) MigrationType() migration.MigrationFSType {
        return migration.MigrationFSType_ZFS
 }
@@ -3325,12 +3200,6 @@ func (s *storageZfs) StoragePoolVolumeCopy(source 
*api.StorageVolumeSource) erro
        return nil
 }
 
-func (s *zfsMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, op 
*operation, bwlimit string, storage storage, volumeOnly bool) error {
-       msg := fmt.Sprintf("Function not implemented")
-       logger.Errorf(msg)
-       return fmt.Errorf(msg)
-}
-
 func (s *storageZfs) StorageMigrationSource(args MigrationSourceArgs) 
(MigrationStorageSourceDriver, error) {
        return rsyncStorageMigrationSource(args)
 }

From 757e7f035825805fafba50fbd5cc68b94135ef04 Mon Sep 17 00:00:00 2001
From: Thomas Hipp <thomas.h...@canonical.com>
Date: Fri, 30 Aug 2019 20:08:11 +0200
Subject: [PATCH 6/6] lxd: Move ceph migration code

The migration code needs to be moved to its own file since
storage_ceph.go will be removed in the future. The code however needs
to stay in the main package (for now) since it depends on the container
interface.

Signed-off-by: Thomas Hipp <thomas.h...@canonical.com>
---
 lxd/storage_ceph.go                 | 256 +++++++++++++++++++
 lxd/storage_ceph_migration.go       | 366 ----------------------------
 lxd/storage_ceph_migration_utils.go | 128 ----------
 lxd/storage_migration_ceph.go       | 225 +++++++++++++++++
 4 files changed, 481 insertions(+), 494 deletions(-)
 delete mode 100644 lxd/storage_ceph_migration.go
 delete mode 100644 lxd/storage_ceph_migration_utils.go
 create mode 100644 lxd/storage_migration_ceph.go

diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go
index ddbc1d7f16..e28fb838b6 100644
--- a/lxd/storage_ceph.go
+++ b/lxd/storage_ceph.go
@@ -7,6 +7,7 @@ import (
        "io"
        "io/ioutil"
        "os"
+       "os/exec"
        "strings"
 
        "github.com/gorilla/websocket"
@@ -14,6 +15,7 @@ import (
        "golang.org/x/sys/unix"
 
        "github.com/lxc/lxd/lxd/db"
+       "github.com/lxc/lxd/lxd/migration"
        "github.com/lxc/lxd/lxd/project"
        driver "github.com/lxc/lxd/lxd/storage"
        "github.com/lxc/lxd/shared"
@@ -2804,3 +2806,257 @@ func (s *storageCeph) 
StoragePoolVolumeSnapshotRename(newName string) error {
 
        return s.s.Cluster.StoragePoolVolumeRename("default", s.volume.Name, 
fullSnapshotName, storagePoolVolumeTypeCustom, s.poolID)
 }
+
+func (s *storageCeph) MigrationType() migration.MigrationFSType {
+       return migration.MigrationFSType_RBD
+}
+
+func (s *storageCeph) PreservesInodes() bool {
+       return false
+}
+
+func (s *storageCeph) MigrationSource(args MigrationSourceArgs) 
(MigrationStorageSourceDriver, error) {
+       // If the container is a snapshot, let's just send that. We don't need
+       // to send anything else, because that's all the user asked for.
+       if args.Container.IsSnapshot() {
+               return &rbdMigrationSourceDriver{
+                       container: args.Container,
+                       ceph:      s,
+               }, nil
+       }
+
+       driver := rbdMigrationSourceDriver{
+               container:        args.Container,
+               snapshots:        []container{},
+               rbdSnapshotNames: []string{},
+               ceph:             s,
+       }
+
+       containerName := args.Container.Name()
+       if args.ContainerOnly {
+               logger.Debugf(`Only migrating the RBD storage volume for 
container "%s" on storage pool "%s`, containerName, s.pool.Name)
+               return &driver, nil
+       }
+
+       // List all the snapshots in order of reverse creation. The idea here is
+       // that we send the oldest to newest snapshot, hopefully saving on xfer
+       // costs. Then, after all that, we send the container itself.
+       snapshots, err := cephRBDVolumeListSnapshots(s.ClusterName,
+               s.OSDPoolName, project.Prefix(args.Container.Project(), 
containerName),
+               storagePoolVolumeTypeNameContainer, s.UserName)
+       if err != nil {
+               if err != db.ErrNoSuchObject {
+                       logger.Errorf(`Failed to list snapshots for RBD storage 
volume "%s" on storage pool "%s": %s`, containerName, s.pool.Name, err)
+                       return nil, err
+               }
+       }
+       logger.Debugf(`Retrieved snapshots "%v" for RBD storage volume "%s" on 
storage pool "%s"`, snapshots, containerName, s.pool.Name)
+
+       for _, snap := range snapshots {
+               // In the case of e.g. multiple copies running at the same time,
+               // we will have potentially multiple migration-send snapshots.
+               // (Or in the case of the test suite, sometimes one will take
+               // too long to delete.)
+               if !strings.HasPrefix(snap, "snapshot_") {
+                       continue
+               }
+
+               lxdName := fmt.Sprintf("%s%s%s", containerName, 
shared.SnapshotDelimiter, snap[len("snapshot_"):])
+               snapshot, err := containerLoadByProjectAndName(s.s, 
args.Container.Project(), lxdName)
+               if err != nil {
+                       logger.Errorf(`Failed to load snapshot "%s" for RBD 
storage volume "%s" on storage pool "%s": %s`, lxdName, containerName, 
s.pool.Name, err)
+                       return nil, err
+               }
+
+               driver.snapshots = append(driver.snapshots, snapshot)
+               driver.rbdSnapshotNames = append(driver.rbdSnapshotNames, snap)
+       }
+
+       return &driver, nil
+}
+
+func (s *storageCeph) MigrationSink(conn *websocket.Conn, op *operation, args 
MigrationSinkArgs) error {
+       // Check that we received a valid root disk device with a pool property
+       // set.
+       parentStoragePool := ""
+       parentExpandedDevices := args.Container.ExpandedDevices()
+       parentLocalRootDiskDeviceKey, parentLocalRootDiskDevice, _ := 
shared.GetRootDiskDevice(parentExpandedDevices)
+       if parentLocalRootDiskDeviceKey != "" {
+               parentStoragePool = parentLocalRootDiskDevice["pool"]
+       }
+
+       // A little neuroticism.
+       if parentStoragePool == "" {
+               return fmt.Errorf(`Detected that the container's root device ` +
+                       `is missing the pool property during RBD migration`)
+       }
+       logger.Debugf(`Detected root disk device with pool property set to "%s" 
during RBD migration`, parentStoragePool)
+
+       // create empty volume for container
+       // TODO: The cluster name can be different between LXD instances. Find
+       // out what to do in this case. Maybe I'm overthinking this and if the
+       // pool exists and we were able to initialize a new storage interface on
+       // the receiving LXD instance it also means that s.ClusterName has been
+       // set to the correct cluster name for that LXD instance. Yeah, I think
+       // that's actually correct.
+       containerName := args.Container.Name()
+       if !cephRBDVolumeExists(s.ClusterName, s.OSDPoolName, 
project.Prefix(args.Container.Project(), containerName), 
storagePoolVolumeTypeNameContainer, s.UserName) {
+               err := cephRBDVolumeCreate(s.ClusterName, s.OSDPoolName, 
project.Prefix(args.Container.Project(), containerName), 
storagePoolVolumeTypeNameContainer, "0", s.UserName)
+               if err != nil {
+                       logger.Errorf(`Failed to create RBD storage volume "%s" 
for cluster "%s" in OSD pool "%s" on storage pool "%s": %s`, containerName, 
s.ClusterName, s.OSDPoolName, s.pool.Name, err)
+                       return err
+               }
+               logger.Debugf(`Created RBD storage volume "%s" on storage pool 
"%s"`, containerName, s.pool.Name)
+       }
+
+       if len(args.Snapshots) > 0 {
+               snapshotMntPointSymlinkTarget := 
shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", 
project.Prefix(args.Container.Project(), containerName))
+               snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(args.Container.Project(), containerName))
+               if !shared.PathExists(snapshotMntPointSymlink) {
+                       err := os.Symlink(snapshotMntPointSymlinkTarget, 
snapshotMntPointSymlink)
+                       if err != nil {
+                               return err
+                       }
+               }
+       }
+
+       // Now we're ready to receive the actual fs.
+       recvName := fmt.Sprintf("%s/container_%s", s.OSDPoolName, 
project.Prefix(args.Container.Project(), containerName))
+       for _, snap := range args.Snapshots {
+               curSnapName := snap.GetName()
+               ctArgs := 
snapshotProtobufToContainerArgs(args.Container.Project(), containerName, snap)
+
+               // Ensure that snapshot and parent container have the same
+               // storage pool in their local root disk device.  If the root
+               // disk device for the snapshot comes from a profile on the new
+               // instance as well we don't need to do anything.
+               if ctArgs.Devices != nil {
+                       snapLocalRootDiskDeviceKey, _, _ := 
shared.GetRootDiskDevice(ctArgs.Devices)
+                       if snapLocalRootDiskDeviceKey != "" {
+                               
ctArgs.Devices[snapLocalRootDiskDeviceKey]["pool"] = parentStoragePool
+                       }
+               }
+               _, err := 
containerCreateEmptySnapshot(args.Container.DaemonState(), ctArgs)
+               if err != nil {
+                       logger.Errorf(`Failed to create empty RBD storage 
volume for container "%s" on storage pool "%s: %s`, containerName, 
s.OSDPoolName, err)
+                       return err
+               }
+               logger.Debugf(`Created empty RBD storage volume for container 
"%s" on storage pool "%s`, containerName, s.OSDPoolName)
+
+               wrapper := StorageProgressWriter(op, "fs_progress", curSnapName)
+               err = s.rbdRecv(conn, recvName, wrapper)
+               if err != nil {
+                       logger.Errorf(`Failed to receive RBD storage volume 
"%s": %s`, curSnapName, err)
+                       return err
+               }
+               logger.Debugf(`Received RBD storage volume "%s"`, curSnapName)
+
+               snapshotMntPoint := 
driver.GetSnapshotMountPoint(args.Container.Project(), s.pool.Name, 
fmt.Sprintf("%s/%s", containerName, *snap.Name))
+               if !shared.PathExists(snapshotMntPoint) {
+                       err := os.MkdirAll(snapshotMntPoint, 0700)
+                       if err != nil {
+                               return err
+                       }
+               }
+       }
+
+       defer func() {
+               snaps, err := cephRBDVolumeListSnapshots(s.ClusterName, 
s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), 
storagePoolVolumeTypeNameContainer, s.UserName)
+               if err == nil {
+                       for _, snap := range snaps {
+                               snapOnlyName, _, _ := 
shared.ContainerGetParentAndSnapshotName(snap)
+                               if !strings.HasPrefix(snapOnlyName, 
"migration-send") {
+                                       continue
+                               }
+
+                               err := cephRBDSnapshotDelete(s.ClusterName, 
s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), 
storagePoolVolumeTypeNameContainer, snapOnlyName, s.UserName)
+                               if err != nil {
+                                       logger.Warnf(`Failed to delete RBD 
container storage for snapshot "%s" of container "%s"`, snapOnlyName, 
containerName)
+                               }
+                       }
+               }
+       }()
+
+       // receive the container itself
+       wrapper := StorageProgressWriter(op, "fs_progress", containerName)
+       err := s.rbdRecv(conn, recvName, wrapper)
+       if err != nil {
+               logger.Errorf(`Failed to receive RBD storage volume "%s": %s`, 
recvName, err)
+               return err
+       }
+       logger.Debugf(`Received RBD storage volume "%s"`, recvName)
+
+       if args.Live {
+               err := s.rbdRecv(conn, recvName, wrapper)
+               if err != nil {
+                       logger.Errorf(`Failed to receive RBD storage volume 
"%s": %s`, recvName, err)
+                       return err
+               }
+               logger.Debugf(`Received RBD storage volume "%s"`, recvName)
+       }
+
+       // Re-generate the UUID
+       err = s.cephRBDGenerateUUID(project.Prefix(args.Container.Project(), 
args.Container.Name()), storagePoolVolumeTypeNameContainer)
+       if err != nil {
+               return err
+       }
+
+       containerMntPoint := 
driver.GetContainerMountPoint(args.Container.Project(), s.pool.Name, 
containerName)
+       err = driver.CreateContainerMountpoint(
+               containerMntPoint,
+               args.Container.Path(),
+               args.Container.IsPrivileged())
+       if err != nil {
+               logger.Errorf(`Failed to create mountpoint "%s" for RBD storage 
volume for container "%s" on storage pool "%s": %s"`, containerMntPoint, 
containerName, s.pool.Name, err)
+               return err
+       }
+       logger.Debugf(`Created mountpoint "%s" for RBD storage volume for 
container "%s" on storage pool "%s""`, containerMntPoint, containerName, 
s.pool.Name)
+
+       return nil
+}
+func (s *storageCeph) rbdRecv(conn *websocket.Conn,
+       volumeName string,
+       writeWrapper func(io.WriteCloser) io.WriteCloser) error {
+       args := []string{
+               "import-diff",
+               "--cluster", s.ClusterName,
+               "-",
+               volumeName,
+       }
+
+       cmd := exec.Command("rbd", args...)
+
+       stdin, err := cmd.StdinPipe()
+       if err != nil {
+               return err
+       }
+
+       stderr, err := cmd.StderrPipe()
+       if err != nil {
+               return err
+       }
+
+       err = cmd.Start()
+       if err != nil {
+               return err
+       }
+
+       writePipe := io.WriteCloser(stdin)
+       if writeWrapper != nil {
+               writePipe = writeWrapper(stdin)
+       }
+
+       <-shared.WebsocketRecvStream(writePipe, conn)
+
+       output, err := ioutil.ReadAll(stderr)
+       if err != nil {
+               logger.Debugf(`Failed to read stderr output from "rbd 
import-diff": %s`, err)
+       }
+
+       err = cmd.Wait()
+       if err != nil {
+               logger.Errorf(`Failed to perform "rbd import-diff": %s`, 
string(output))
+       }
+
+       return err
+}
diff --git a/lxd/storage_ceph_migration.go b/lxd/storage_ceph_migration.go
deleted file mode 100644
index 57e7839ef1..0000000000
--- a/lxd/storage_ceph_migration.go
+++ /dev/null
@@ -1,366 +0,0 @@
-package main
-
-import (
-       "fmt"
-       "os"
-       "strings"
-
-       "github.com/gorilla/websocket"
-       "github.com/pborman/uuid"
-
-       "github.com/lxc/lxd/lxd/db"
-       "github.com/lxc/lxd/lxd/migration"
-       "github.com/lxc/lxd/lxd/project"
-       driver "github.com/lxc/lxd/lxd/storage"
-       "github.com/lxc/lxd/shared"
-       "github.com/lxc/lxd/shared/logger"
-)
-
-type rbdMigrationSourceDriver struct {
-       container        container
-       snapshots        []container
-       rbdSnapshotNames []string
-       ceph             *storageCeph
-       runningSnapName  string
-       stoppedSnapName  string
-}
-
-func (s *rbdMigrationSourceDriver) Snapshots() []container {
-       return s.snapshots
-}
-
-func (s *rbdMigrationSourceDriver) Cleanup() {
-       containerName := s.container.Name()
-
-       if s.stoppedSnapName != "" {
-               err := cephRBDSnapshotDelete(s.ceph.ClusterName, 
s.ceph.OSDPoolName,
-                       project.Prefix(s.container.Project(), containerName), 
storagePoolVolumeTypeNameContainer,
-                       s.stoppedSnapName, s.ceph.UserName)
-               if err != nil {
-                       logger.Warnf(`Failed to delete RBD snapshot "%s" of 
container "%s"`, s.stoppedSnapName, containerName)
-               }
-       }
-
-       if s.runningSnapName != "" {
-               err := cephRBDSnapshotDelete(s.ceph.ClusterName, 
s.ceph.OSDPoolName,
-                       project.Prefix(s.container.Project(), containerName), 
storagePoolVolumeTypeNameContainer,
-                       s.runningSnapName, s.ceph.UserName)
-               if err != nil {
-                       logger.Warnf(`Failed to delete RBD snapshot "%s" of 
container "%s"`, s.runningSnapName, containerName)
-               }
-       }
-}
-
-func (s *rbdMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, 
bwlimit string) error {
-       containerName := s.container.Name()
-       s.stoppedSnapName = fmt.Sprintf("migration-send-%s", 
uuid.NewRandom().String())
-       err := cephRBDSnapshotCreate(s.ceph.ClusterName, s.ceph.OSDPoolName,
-               project.Prefix(s.container.Project(), containerName), 
storagePoolVolumeTypeNameContainer,
-               s.stoppedSnapName, s.ceph.UserName)
-       if err != nil {
-               logger.Errorf(`Failed to create snapshot "%s" for RBD storage 
volume for image "%s" on storage pool "%s": %s`, s.stoppedSnapName, 
containerName, s.ceph.pool.Name, err)
-               return err
-       }
-
-       cur := fmt.Sprintf("%s/container_%s@%s", s.ceph.OSDPoolName,
-               project.Prefix(s.container.Project(), containerName), 
s.stoppedSnapName)
-       err = s.rbdSend(conn, cur, s.runningSnapName, nil)
-       if err != nil {
-               logger.Errorf(`Failed to send exported diff of RBD storage 
volume "%s" from snapshot "%s": %s`, cur, s.runningSnapName, err)
-               return err
-       }
-       logger.Debugf(`Sent exported diff of RBD storage volume "%s" from 
snapshot "%s"`, cur, s.stoppedSnapName)
-
-       return nil
-}
-
-func (s *rbdMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn,
-       op *operation, bwlimit string, containerOnly bool) error {
-       containerName := s.container.Name()
-       if s.container.IsSnapshot() {
-               // ContainerSnapshotStart() will create the clone that is
-               // referenced by sendName here.
-               containerOnlyName, snapOnlyName, _ := 
shared.ContainerGetParentAndSnapshotName(containerName)
-               sendName := fmt.Sprintf(
-                       "%s/snapshots_%s_%s_start_clone",
-                       s.ceph.OSDPoolName,
-                       containerOnlyName,
-                       snapOnlyName)
-               wrapper := StorageProgressReader(op, "fs_progress", 
containerName)
-
-               err := s.rbdSend(conn, sendName, "", wrapper)
-               if err != nil {
-                       logger.Errorf(`Failed to send RBD storage volume "%s": 
%s`, sendName, err)
-                       return err
-               }
-               logger.Debugf(`Sent RBD storage volume "%s"`, sendName)
-
-               return nil
-       }
-
-       lastSnap := ""
-       if !containerOnly {
-               for i, snap := range s.rbdSnapshotNames {
-                       prev := ""
-                       if i > 0 {
-                               prev = s.rbdSnapshotNames[i-1]
-                       }
-
-                       lastSnap = snap
-
-                       sendSnapName := fmt.Sprintf(
-                               "%s/container_%s@%s",
-                               s.ceph.OSDPoolName,
-                               project.Prefix(s.container.Project(), 
containerName),
-                               snap)
-
-                       wrapper := StorageProgressReader(op, "fs_progress", 
snap)
-
-                       err := s.rbdSend(
-                               conn,
-                               sendSnapName,
-                               prev,
-                               wrapper)
-                       if err != nil {
-                               logger.Errorf(`Failed to send exported diff of 
RBD storage volume "%s" from snapshot "%s": %s`, sendSnapName, prev, err)
-                               return err
-                       }
-                       logger.Debugf(`Sent exported diff of RBD storage volume 
"%s" from snapshot "%s"`, sendSnapName, prev)
-               }
-       }
-
-       s.runningSnapName = fmt.Sprintf("migration-send-%s", 
uuid.NewRandom().String())
-       err := cephRBDSnapshotCreate(s.ceph.ClusterName, s.ceph.OSDPoolName,
-               project.Prefix(s.container.Project(), containerName), 
storagePoolVolumeTypeNameContainer,
-               s.runningSnapName, s.ceph.UserName)
-       if err != nil {
-               logger.Errorf(`Failed to create snapshot "%s" for RBD storage 
volume for image "%s" on storage pool "%s": %s`, s.runningSnapName, 
containerName, s.ceph.pool.Name, err)
-               return err
-       }
-
-       cur := fmt.Sprintf("%s/container_%s@%s", s.ceph.OSDPoolName,
-               project.Prefix(s.container.Project(), containerName), 
s.runningSnapName)
-       wrapper := StorageProgressReader(op, "fs_progress", containerName)
-       err = s.rbdSend(conn, cur, lastSnap, wrapper)
-       if err != nil {
-               logger.Errorf(`Failed to send exported diff of RBD storage 
volume "%s" from snapshot "%s": %s`, s.runningSnapName, lastSnap, err)
-               return err
-       }
-       logger.Debugf(`Sent exported diff of RBD storage volume "%s" from 
snapshot "%s"`, s.runningSnapName, lastSnap)
-
-       return nil
-}
-
-func (s *rbdMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, op 
*operation, bwlimit string, storage storage, volumeOnly bool) error {
-       msg := fmt.Sprintf("Function not implemented")
-       logger.Errorf(msg)
-       return fmt.Errorf(msg)
-}
-
-func (s *storageCeph) MigrationType() migration.MigrationFSType {
-       return migration.MigrationFSType_RBD
-}
-
-func (s *storageCeph) PreservesInodes() bool {
-       return false
-}
-
-func (s *storageCeph) MigrationSource(args MigrationSourceArgs) 
(MigrationStorageSourceDriver, error) {
-       // If the container is a snapshot, let's just send that. We don't need
-       // to send anything else, because that's all the user asked for.
-       if args.Container.IsSnapshot() {
-               return &rbdMigrationSourceDriver{
-                       container: args.Container,
-                       ceph:      s,
-               }, nil
-       }
-
-       driver := rbdMigrationSourceDriver{
-               container:        args.Container,
-               snapshots:        []container{},
-               rbdSnapshotNames: []string{},
-               ceph:             s,
-       }
-
-       containerName := args.Container.Name()
-       if args.ContainerOnly {
-               logger.Debugf(`Only migrating the RBD storage volume for 
container "%s" on storage pool "%s`, containerName, s.pool.Name)
-               return &driver, nil
-       }
-
-       // List all the snapshots in order of reverse creation. The idea here is
-       // that we send the oldest to newest snapshot, hopefully saving on xfer
-       // costs. Then, after all that, we send the container itself.
-       snapshots, err := cephRBDVolumeListSnapshots(s.ClusterName,
-               s.OSDPoolName, project.Prefix(args.Container.Project(), 
containerName),
-               storagePoolVolumeTypeNameContainer, s.UserName)
-       if err != nil {
-               if err != db.ErrNoSuchObject {
-                       logger.Errorf(`Failed to list snapshots for RBD storage 
volume "%s" on storage pool "%s": %s`, containerName, s.pool.Name, err)
-                       return nil, err
-               }
-       }
-       logger.Debugf(`Retrieved snapshots "%v" for RBD storage volume "%s" on 
storage pool "%s"`, snapshots, containerName, s.pool.Name)
-
-       for _, snap := range snapshots {
-               // In the case of e.g. multiple copies running at the same time,
-               // we will have potentially multiple migration-send snapshots.
-               // (Or in the case of the test suite, sometimes one will take
-               // too long to delete.)
-               if !strings.HasPrefix(snap, "snapshot_") {
-                       continue
-               }
-
-               lxdName := fmt.Sprintf("%s%s%s", containerName, 
shared.SnapshotDelimiter, snap[len("snapshot_"):])
-               snapshot, err := containerLoadByProjectAndName(s.s, 
args.Container.Project(), lxdName)
-               if err != nil {
-                       logger.Errorf(`Failed to load snapshot "%s" for RBD 
storage volume "%s" on storage pool "%s": %s`, lxdName, containerName, 
s.pool.Name, err)
-                       return nil, err
-               }
-
-               driver.snapshots = append(driver.snapshots, snapshot)
-               driver.rbdSnapshotNames = append(driver.rbdSnapshotNames, snap)
-       }
-
-       return &driver, nil
-}
-
-func (s *storageCeph) MigrationSink(conn *websocket.Conn, op *operation, args 
MigrationSinkArgs) error {
-       // Check that we received a valid root disk device with a pool property
-       // set.
-       parentStoragePool := ""
-       parentExpandedDevices := args.Container.ExpandedDevices()
-       parentLocalRootDiskDeviceKey, parentLocalRootDiskDevice, _ := 
shared.GetRootDiskDevice(parentExpandedDevices)
-       if parentLocalRootDiskDeviceKey != "" {
-               parentStoragePool = parentLocalRootDiskDevice["pool"]
-       }
-
-       // A little neuroticism.
-       if parentStoragePool == "" {
-               return fmt.Errorf(`Detected that the container's root device ` +
-                       `is missing the pool property during RBD migration`)
-       }
-       logger.Debugf(`Detected root disk device with pool property set to "%s" 
during RBD migration`, parentStoragePool)
-
-       // create empty volume for container
-       // TODO: The cluster name can be different between LXD instances. Find
-       // out what to do in this case. Maybe I'm overthinking this and if the
-       // pool exists and we were able to initialize a new storage interface on
-       // the receiving LXD instance it also means that s.ClusterName has been
-       // set to the correct cluster name for that LXD instance. Yeah, I think
-       // that's actually correct.
-       containerName := args.Container.Name()
-       if !cephRBDVolumeExists(s.ClusterName, s.OSDPoolName, 
project.Prefix(args.Container.Project(), containerName), 
storagePoolVolumeTypeNameContainer, s.UserName) {
-               err := cephRBDVolumeCreate(s.ClusterName, s.OSDPoolName, 
project.Prefix(args.Container.Project(), containerName), 
storagePoolVolumeTypeNameContainer, "0", s.UserName)
-               if err != nil {
-                       logger.Errorf(`Failed to create RBD storage volume "%s" 
for cluster "%s" in OSD pool "%s" on storage pool "%s": %s`, containerName, 
s.ClusterName, s.OSDPoolName, s.pool.Name, err)
-                       return err
-               }
-               logger.Debugf(`Created RBD storage volume "%s" on storage pool 
"%s"`, containerName, s.pool.Name)
-       }
-
-       if len(args.Snapshots) > 0 {
-               snapshotMntPointSymlinkTarget := 
shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", 
project.Prefix(args.Container.Project(), containerName))
-               snapshotMntPointSymlink := shared.VarPath("snapshots", 
project.Prefix(args.Container.Project(), containerName))
-               if !shared.PathExists(snapshotMntPointSymlink) {
-                       err := os.Symlink(snapshotMntPointSymlinkTarget, 
snapshotMntPointSymlink)
-                       if err != nil {
-                               return err
-                       }
-               }
-       }
-
-       // Now we're ready to receive the actual fs.
-       recvName := fmt.Sprintf("%s/container_%s", s.OSDPoolName, 
project.Prefix(args.Container.Project(), containerName))
-       for _, snap := range args.Snapshots {
-               curSnapName := snap.GetName()
-               ctArgs := 
snapshotProtobufToContainerArgs(args.Container.Project(), containerName, snap)
-
-               // Ensure that snapshot and parent container have the same
-               // storage pool in their local root disk device.  If the root
-               // disk device for the snapshot comes from a profile on the new
-               // instance as well we don't need to do anything.
-               if ctArgs.Devices != nil {
-                       snapLocalRootDiskDeviceKey, _, _ := 
shared.GetRootDiskDevice(ctArgs.Devices)
-                       if snapLocalRootDiskDeviceKey != "" {
-                               
ctArgs.Devices[snapLocalRootDiskDeviceKey]["pool"] = parentStoragePool
-                       }
-               }
-               _, err := 
containerCreateEmptySnapshot(args.Container.DaemonState(), ctArgs)
-               if err != nil {
-                       logger.Errorf(`Failed to create empty RBD storage 
volume for container "%s" on storage pool "%s: %s`, containerName, 
s.OSDPoolName, err)
-                       return err
-               }
-               logger.Debugf(`Created empty RBD storage volume for container 
"%s" on storage pool "%s`, containerName, s.OSDPoolName)
-
-               wrapper := StorageProgressWriter(op, "fs_progress", curSnapName)
-               err = s.rbdRecv(conn, recvName, wrapper)
-               if err != nil {
-                       logger.Errorf(`Failed to receive RBD storage volume 
"%s": %s`, curSnapName, err)
-                       return err
-               }
-               logger.Debugf(`Received RBD storage volume "%s"`, curSnapName)
-
-               snapshotMntPoint := 
driver.GetSnapshotMountPoint(args.Container.Project(), s.pool.Name, 
fmt.Sprintf("%s/%s", containerName, *snap.Name))
-               if !shared.PathExists(snapshotMntPoint) {
-                       err := os.MkdirAll(snapshotMntPoint, 0700)
-                       if err != nil {
-                               return err
-                       }
-               }
-       }
-
-       defer func() {
-               snaps, err := cephRBDVolumeListSnapshots(s.ClusterName, 
s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), 
storagePoolVolumeTypeNameContainer, s.UserName)
-               if err == nil {
-                       for _, snap := range snaps {
-                               snapOnlyName, _, _ := 
shared.ContainerGetParentAndSnapshotName(snap)
-                               if !strings.HasPrefix(snapOnlyName, 
"migration-send") {
-                                       continue
-                               }
-
-                               err := cephRBDSnapshotDelete(s.ClusterName, 
s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), 
storagePoolVolumeTypeNameContainer, snapOnlyName, s.UserName)
-                               if err != nil {
-                                       logger.Warnf(`Failed to delete RBD 
container storage for snapshot "%s" of container "%s"`, snapOnlyName, 
containerName)
-                               }
-                       }
-               }
-       }()
-
-       // receive the container itself
-       wrapper := StorageProgressWriter(op, "fs_progress", containerName)
-       err := s.rbdRecv(conn, recvName, wrapper)
-       if err != nil {
-               logger.Errorf(`Failed to receive RBD storage volume "%s": %s`, 
recvName, err)
-               return err
-       }
-       logger.Debugf(`Received RBD storage volume "%s"`, recvName)
-
-       if args.Live {
-               err := s.rbdRecv(conn, recvName, wrapper)
-               if err != nil {
-                       logger.Errorf(`Failed to receive RBD storage volume 
"%s": %s`, recvName, err)
-                       return err
-               }
-               logger.Debugf(`Received RBD storage volume "%s"`, recvName)
-       }
-
-       // Re-generate the UUID
-       err = s.cephRBDGenerateUUID(project.Prefix(args.Container.Project(), 
args.Container.Name()), storagePoolVolumeTypeNameContainer)
-       if err != nil {
-               return err
-       }
-
-       containerMntPoint := 
driver.GetContainerMountPoint(args.Container.Project(), s.pool.Name, 
containerName)
-       err = driver.CreateContainerMountpoint(
-               containerMntPoint,
-               args.Container.Path(),
-               args.Container.IsPrivileged())
-       if err != nil {
-               logger.Errorf(`Failed to create mountpoint "%s" for RBD storage 
volume for container "%s" on storage pool "%s": %s"`, containerMntPoint, 
containerName, s.pool.Name, err)
-               return err
-       }
-       logger.Debugf(`Created mountpoint "%s" for RBD storage volume for 
container "%s" on storage pool "%s""`, containerMntPoint, containerName, 
s.pool.Name)
-
-       return nil
-}
diff --git a/lxd/storage_ceph_migration_utils.go 
b/lxd/storage_ceph_migration_utils.go
deleted file mode 100644
index 77caee4d8d..0000000000
--- a/lxd/storage_ceph_migration_utils.go
+++ /dev/null
@@ -1,128 +0,0 @@
-package main
-
-import (
-       "io"
-       "io/ioutil"
-       "os/exec"
-
-       "github.com/gorilla/websocket"
-
-       "github.com/lxc/lxd/shared"
-       "github.com/lxc/lxd/shared/logger"
-)
-
-// Let's say we want to send the a container "a" including snapshots "snap0" 
and
-// "snap1" on storage pool "pool1" from LXD "l1" to LXD "l2" on storage pool
-// "pool2":
-//
-// The pool layout on "l1" would be:
-//     pool1/container_a
-//     pool1/container_a@snapshot_snap0
-//     pool1/container_a@snapshot_snap1
-//
-// Then we need to send:
-//     rbd export-diff pool1/container_a@snapshot_snap0 - | rbd import-diff - 
pool2/container_a
-// (Note that pool2/container_a must have been created by the receiving LXD
-// instance before.)
-//     rbd export-diff pool1/container_a@snapshot_snap1 --from-snap 
snapshot_snap0 - | rbd import-diff - pool2/container_a
-//     rbd export-diff pool1/container_a --from-snap snapshot_snap1 - | rbd 
import-diff - pool2/container_a
-func (s *rbdMigrationSourceDriver) rbdSend(conn *websocket.Conn,
-       volumeName string,
-       volumeParentName string,
-       readWrapper func(io.ReadCloser) io.ReadCloser) error {
-       args := []string{
-               "export-diff",
-               "--cluster", s.ceph.ClusterName,
-               volumeName,
-       }
-
-       if volumeParentName != "" {
-               args = append(args, "--from-snap", volumeParentName)
-       }
-
-       // redirect output to stdout
-       args = append(args, "-")
-
-       cmd := exec.Command("rbd", args...)
-
-       stdout, err := cmd.StdoutPipe()
-       if err != nil {
-               return err
-       }
-
-       readPipe := io.ReadCloser(stdout)
-       if readWrapper != nil {
-               readPipe = readWrapper(stdout)
-       }
-
-       stderr, err := cmd.StderrPipe()
-       if err != nil {
-               return err
-       }
-
-       err = cmd.Start()
-       if err != nil {
-               return err
-       }
-
-       <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024)
-
-       output, err := ioutil.ReadAll(stderr)
-       if err != nil {
-               logger.Debugf(`Failed to read stderr output from "rbd 
export-diff": %s`, err)
-       }
-
-       err = cmd.Wait()
-       if err != nil {
-               logger.Errorf(`Failed to perform "rbd export-diff": %s`, 
string(output))
-       }
-
-       return err
-}
-
-func (s *storageCeph) rbdRecv(conn *websocket.Conn,
-       volumeName string,
-       writeWrapper func(io.WriteCloser) io.WriteCloser) error {
-       args := []string{
-               "import-diff",
-               "--cluster", s.ClusterName,
-               "-",
-               volumeName,
-       }
-
-       cmd := exec.Command("rbd", args...)
-
-       stdin, err := cmd.StdinPipe()
-       if err != nil {
-               return err
-       }
-
-       stderr, err := cmd.StderrPipe()
-       if err != nil {
-               return err
-       }
-
-       err = cmd.Start()
-       if err != nil {
-               return err
-       }
-
-       writePipe := io.WriteCloser(stdin)
-       if writeWrapper != nil {
-               writePipe = writeWrapper(stdin)
-       }
-
-       <-shared.WebsocketRecvStream(writePipe, conn)
-
-       output, err := ioutil.ReadAll(stderr)
-       if err != nil {
-               logger.Debugf(`Failed to read stderr output from "rbd 
import-diff": %s`, err)
-       }
-
-       err = cmd.Wait()
-       if err != nil {
-               logger.Errorf(`Failed to perform "rbd import-diff": %s`, 
string(output))
-       }
-
-       return err
-}
diff --git a/lxd/storage_migration_ceph.go b/lxd/storage_migration_ceph.go
new file mode 100644
index 0000000000..83cc7cfc0c
--- /dev/null
+++ b/lxd/storage_migration_ceph.go
@@ -0,0 +1,225 @@
+package main
+
+import (
+       "fmt"
+       "io"
+       "io/ioutil"
+       "os/exec"
+
+       "github.com/gorilla/websocket"
+       "github.com/pborman/uuid"
+
+       "github.com/lxc/lxd/lxd/project"
+       "github.com/lxc/lxd/shared"
+       "github.com/lxc/lxd/shared/logger"
+)
+
+type rbdMigrationSourceDriver struct {
+       container        container
+       snapshots        []container
+       rbdSnapshotNames []string
+       ceph             *storageCeph
+       runningSnapName  string
+       stoppedSnapName  string
+}
+
+func (s *rbdMigrationSourceDriver) Snapshots() []container {
+       return s.snapshots
+}
+
+func (s *rbdMigrationSourceDriver) Cleanup() {
+       containerName := s.container.Name()
+
+       if s.stoppedSnapName != "" {
+               err := cephRBDSnapshotDelete(s.ceph.ClusterName, 
s.ceph.OSDPoolName,
+                       project.Prefix(s.container.Project(), containerName), 
storagePoolVolumeTypeNameContainer,
+                       s.stoppedSnapName, s.ceph.UserName)
+               if err != nil {
+                       logger.Warnf(`Failed to delete RBD snapshot "%s" of 
container "%s"`, s.stoppedSnapName, containerName)
+               }
+       }
+
+       if s.runningSnapName != "" {
+               err := cephRBDSnapshotDelete(s.ceph.ClusterName, 
s.ceph.OSDPoolName,
+                       project.Prefix(s.container.Project(), containerName), 
storagePoolVolumeTypeNameContainer,
+                       s.runningSnapName, s.ceph.UserName)
+               if err != nil {
+                       logger.Warnf(`Failed to delete RBD snapshot "%s" of 
container "%s"`, s.runningSnapName, containerName)
+               }
+       }
+}
+
+func (s *rbdMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, 
bwlimit string) error {
+       containerName := s.container.Name()
+       s.stoppedSnapName = fmt.Sprintf("migration-send-%s", 
uuid.NewRandom().String())
+       err := cephRBDSnapshotCreate(s.ceph.ClusterName, s.ceph.OSDPoolName,
+               project.Prefix(s.container.Project(), containerName), 
storagePoolVolumeTypeNameContainer,
+               s.stoppedSnapName, s.ceph.UserName)
+       if err != nil {
+               logger.Errorf(`Failed to create snapshot "%s" for RBD storage 
volume for image "%s" on storage pool "%s": %s`, s.stoppedSnapName, 
containerName, s.ceph.pool.Name, err)
+               return err
+       }
+
+       cur := fmt.Sprintf("%s/container_%s@%s", s.ceph.OSDPoolName,
+               project.Prefix(s.container.Project(), containerName), 
s.stoppedSnapName)
+       err = s.rbdSend(conn, cur, s.runningSnapName, nil)
+       if err != nil {
+               logger.Errorf(`Failed to send exported diff of RBD storage 
volume "%s" from snapshot "%s": %s`, cur, s.runningSnapName, err)
+               return err
+       }
+       logger.Debugf(`Sent exported diff of RBD storage volume "%s" from 
snapshot "%s"`, cur, s.stoppedSnapName)
+
+       return nil
+}
+
+func (s *rbdMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn,
+       op *operation, bwlimit string, containerOnly bool) error {
+       containerName := s.container.Name()
+       if s.container.IsSnapshot() {
+               // ContainerSnapshotStart() will create the clone that is
+               // referenced by sendName here.
+               containerOnlyName, snapOnlyName, _ := 
shared.ContainerGetParentAndSnapshotName(containerName)
+               sendName := fmt.Sprintf(
+                       "%s/snapshots_%s_%s_start_clone",
+                       s.ceph.OSDPoolName,
+                       containerOnlyName,
+                       snapOnlyName)
+               wrapper := StorageProgressReader(op, "fs_progress", 
containerName)
+
+               err := s.rbdSend(conn, sendName, "", wrapper)
+               if err != nil {
+                       logger.Errorf(`Failed to send RBD storage volume "%s": 
%s`, sendName, err)
+                       return err
+               }
+               logger.Debugf(`Sent RBD storage volume "%s"`, sendName)
+
+               return nil
+       }
+
+       lastSnap := ""
+       if !containerOnly {
+               for i, snap := range s.rbdSnapshotNames {
+                       prev := ""
+                       if i > 0 {
+                               prev = s.rbdSnapshotNames[i-1]
+                       }
+
+                       lastSnap = snap
+
+                       sendSnapName := fmt.Sprintf(
+                               "%s/container_%s@%s",
+                               s.ceph.OSDPoolName,
+                               project.Prefix(s.container.Project(), 
containerName),
+                               snap)
+
+                       wrapper := StorageProgressReader(op, "fs_progress", 
snap)
+
+                       err := s.rbdSend(
+                               conn,
+                               sendSnapName,
+                               prev,
+                               wrapper)
+                       if err != nil {
+                               logger.Errorf(`Failed to send exported diff of 
RBD storage volume "%s" from snapshot "%s": %s`, sendSnapName, prev, err)
+                               return err
+                       }
+                       logger.Debugf(`Sent exported diff of RBD storage volume 
"%s" from snapshot "%s"`, sendSnapName, prev)
+               }
+       }
+
+       s.runningSnapName = fmt.Sprintf("migration-send-%s", 
uuid.NewRandom().String())
+       err := cephRBDSnapshotCreate(s.ceph.ClusterName, s.ceph.OSDPoolName,
+               project.Prefix(s.container.Project(), containerName), 
storagePoolVolumeTypeNameContainer,
+               s.runningSnapName, s.ceph.UserName)
+       if err != nil {
+               logger.Errorf(`Failed to create snapshot "%s" for RBD storage 
volume for image "%s" on storage pool "%s": %s`, s.runningSnapName, 
containerName, s.ceph.pool.Name, err)
+               return err
+       }
+
+       cur := fmt.Sprintf("%s/container_%s@%s", s.ceph.OSDPoolName,
+               project.Prefix(s.container.Project(), containerName), 
s.runningSnapName)
+       wrapper := StorageProgressReader(op, "fs_progress", containerName)
+       err = s.rbdSend(conn, cur, lastSnap, wrapper)
+       if err != nil {
+               logger.Errorf(`Failed to send exported diff of RBD storage 
volume "%s" from snapshot "%s": %s`, s.runningSnapName, lastSnap, err)
+               return err
+       }
+       logger.Debugf(`Sent exported diff of RBD storage volume "%s" from 
snapshot "%s"`, s.runningSnapName, lastSnap)
+
+       return nil
+}
+
+func (s *rbdMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, op 
*operation, bwlimit string, storage storage, volumeOnly bool) error {
+       msg := fmt.Sprintf("Function not implemented")
+       logger.Errorf(msg)
+       return fmt.Errorf(msg)
+}
+
+// Let's say we want to send the a container "a" including snapshots "snap0" 
and
+// "snap1" on storage pool "pool1" from LXD "l1" to LXD "l2" on storage pool
+// "pool2":
+//
+// The pool layout on "l1" would be:
+//     pool1/container_a
+//     pool1/container_a@snapshot_snap0
+//     pool1/container_a@snapshot_snap1
+//
+// Then we need to send:
+//     rbd export-diff pool1/container_a@snapshot_snap0 - | rbd import-diff - 
pool2/container_a
+// (Note that pool2/container_a must have been created by the receiving LXD
+// instance before.)
+//     rbd export-diff pool1/container_a@snapshot_snap1 --from-snap 
snapshot_snap0 - | rbd import-diff - pool2/container_a
+//     rbd export-diff pool1/container_a --from-snap snapshot_snap1 - | rbd 
import-diff - pool2/container_a
+func (s *rbdMigrationSourceDriver) rbdSend(conn *websocket.Conn,
+       volumeName string,
+       volumeParentName string,
+       readWrapper func(io.ReadCloser) io.ReadCloser) error {
+       args := []string{
+               "export-diff",
+               "--cluster", s.ceph.ClusterName,
+               volumeName,
+       }
+
+       if volumeParentName != "" {
+               args = append(args, "--from-snap", volumeParentName)
+       }
+
+       // redirect output to stdout
+       args = append(args, "-")
+
+       cmd := exec.Command("rbd", args...)
+
+       stdout, err := cmd.StdoutPipe()
+       if err != nil {
+               return err
+       }
+
+       readPipe := io.ReadCloser(stdout)
+       if readWrapper != nil {
+               readPipe = readWrapper(stdout)
+       }
+
+       stderr, err := cmd.StderrPipe()
+       if err != nil {
+               return err
+       }
+
+       err = cmd.Start()
+       if err != nil {
+               return err
+       }
+
+       <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024)
+
+       output, err := ioutil.ReadAll(stderr)
+       if err != nil {
+               logger.Debugf(`Failed to read stderr output from "rbd 
export-diff": %s`, err)
+       }
+
+       err = cmd.Wait()
+       if err != nil {
+               logger.Errorf(`Failed to perform "rbd export-diff": %s`, 
string(output))
+       }
+
+       return err
+}
_______________________________________________
lxc-devel mailing list
lxc-devel@lists.linuxcontainers.org
http://lists.linuxcontainers.org/listinfo/lxc-devel

Reply via email to