The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/6136
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) ===
From 8b173cae21615d224c960b82741a8615f3bff581 Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Mon, 19 Aug 2019 19:36:04 +0200 Subject: [PATCH 1/6] lxd: Move Create{Container,Snapshot}Mountpoint to storage Signed-off-by: Thomas Hipp <thomas.h...@canonical.com> --- lxd/api_internal.go | 4 +-- lxd/container_post.go | 4 +-- lxd/patches.go | 10 +++--- lxd/storage.go | 55 -------------------------------- lxd/storage/storage.go | 60 +++++++++++++++++++++++++++++++++++ lxd/storage_btrfs.go | 16 +++++----- lxd/storage_ceph.go | 6 ++-- lxd/storage_ceph_migration.go | 2 +- lxd/storage_ceph_utils.go | 8 ++--- lxd/storage_dir.go | 14 ++++---- lxd/storage_lvm.go | 10 +++--- lxd/storage_lvm_utils.go | 8 ++--- lxd/storage_zfs.go | 14 ++++---- lxd/storage_zfs_utils.go | 4 +-- 14 files changed, 110 insertions(+), 105 deletions(-) diff --git a/lxd/api_internal.go b/lxd/api_internal.go index ddfffe5ae8..cbe27ad22e 100644 --- a/lxd/api_internal.go +++ b/lxd/api_internal.go @@ -923,7 +923,7 @@ func internalImport(d *Daemon, r *http.Request) Response { if backup.Container.Config["security.privileged"] == "" { isPrivileged = true } - err = createContainerMountpoint(containerMntPoint, containerPath, + err = driver.CreateContainerMountpoint(containerMntPoint, containerPath, isPrivileged) if err != nil { return InternalError(err) @@ -1029,7 +1029,7 @@ func internalImport(d *Daemon, r *http.Request) Response { sourceName = project.Prefix(projectName, sourceName) snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", backup.Pool.Name, "containers-snapshots", sourceName) snapshotMntPointSymlink := shared.VarPath("snapshots", sourceName) - err = createSnapshotMountpoint(snapshotMountPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) + err = driver.CreateSnapshotMountpoint(snapshotMountPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) if err != nil { return InternalError(err) } diff --git a/lxd/container_post.go b/lxd/container_post.go index 5883dc69c9..8bd33b2dfa 100644 --- a/lxd/container_post.go +++ b/lxd/container_post.go @@ -531,7 +531,7 @@ func containerPostCreateContainerMountPoint(d *Daemon, project, containerName st } containerMntPoint := driver.GetContainerMountPoint(c.Project(), poolName, containerName) - err = createContainerMountpoint(containerMntPoint, c.Path(), c.IsPrivileged()) + err = driver.CreateContainerMountpoint(containerMntPoint, c.Path(), c.IsPrivileged()) if err != nil { return errors.Wrap(err, "Failed to create container mount point on target node") } @@ -541,7 +541,7 @@ func containerPostCreateContainerMountPoint(d *Daemon, project, containerName st snapshotsSymlinkTarget := shared.VarPath("storage-pools", poolName, "containers-snapshots", containerName) snapshotMntPointSymlink := shared.VarPath("snapshots", containerName) - err := createSnapshotMountpoint(mntPoint, snapshotsSymlinkTarget, snapshotMntPointSymlink) + err := driver.CreateSnapshotMountpoint(mntPoint, snapshotsSymlinkTarget, snapshotMntPointSymlink) if err != nil { return errors.Wrap(err, "Failed to create snapshot mount point on target node") } diff --git a/lxd/patches.go b/lxd/patches.go index 0ee421fa12..4ba86582f7 100644 --- a/lxd/patches.go +++ b/lxd/patches.go @@ -524,7 +524,7 @@ func upgradeFromStorageTypeBtrfs(name string, d *Daemon, defaultPoolName string, // ${LXD_DIR}/containers/<container_name> to // ${LXD_DIR}/storage-pools/<pool>/containers/<container_name> doesntMatter := false - err = createContainerMountpoint(newContainerMntPoint, oldContainerMntPoint, doesntMatter) + err = driver.CreateContainerMountpoint(newContainerMntPoint, oldContainerMntPoint, doesntMatter) if err != nil { return err } @@ -808,7 +808,7 @@ func upgradeFromStorageTypeDir(name string, d *Daemon, defaultPoolName string, d } doesntMatter := false - err = createContainerMountpoint(newContainerMntPoint, oldContainerMntPoint, doesntMatter) + err = driver.CreateContainerMountpoint(newContainerMntPoint, oldContainerMntPoint, doesntMatter) if err != nil { return err } @@ -855,7 +855,7 @@ func upgradeFromStorageTypeDir(name string, d *Daemon, defaultPoolName string, d } // Create a symlink for this container. snapshots. - err = createSnapshotMountpoint(newSnapshotMntPoint, newSnapshotMntPoint, oldSnapshotMntPoint) + err = driver.CreateSnapshotMountpoint(newSnapshotMntPoint, newSnapshotMntPoint, oldSnapshotMntPoint) if err != nil { return err } @@ -1191,7 +1191,7 @@ func upgradeFromStorageTypeLvm(name string, d *Daemon, defaultPoolName string, d // Create the new container mountpoint. doesntMatter := false - err = createContainerMountpoint(newContainerMntPoint, oldContainerMntPoint, doesntMatter) + err = driver.CreateContainerMountpoint(newContainerMntPoint, oldContainerMntPoint, doesntMatter) if err != nil { logger.Errorf("Failed to create container mountpoint \"%s\" for LVM logical volume: %s", newContainerMntPoint, err) return err @@ -1632,7 +1632,7 @@ func upgradeFromStorageTypeZfs(name string, d *Daemon, defaultPoolName string, d // the path but in case it somehow didn't let's do it ourselves. doesntMatter := false newContainerMntPoint := driver.GetContainerMountPoint("default", poolName, ct) - err = createContainerMountpoint(newContainerMntPoint, oldContainerMntPoint, doesntMatter) + err = driver.CreateContainerMountpoint(newContainerMntPoint, oldContainerMntPoint, doesntMatter) if err != nil { logger.Warnf("Failed to create mountpoint for the container: %s", newContainerMntPoint) failedUpgradeEntities = append(failedUpgradeEntities, fmt.Sprintf("containers/%s: Failed to create container mountpoint: %s", ct, err)) diff --git a/lxd/storage.go b/lxd/storage.go index 5ee427f4e8..1a03e2fb56 100644 --- a/lxd/storage.go +++ b/lxd/storage.go @@ -607,40 +607,6 @@ func storagePoolVolumeContainerLoadInit(s *state.State, project, containerName s return storagePoolVolumeInit(s, project, poolName, containerName, storagePoolVolumeTypeContainer) } -func createContainerMountpoint(mountPoint string, mountPointSymlink string, privileged bool) error { - var mode os.FileMode - if privileged { - mode = 0700 - } else { - mode = 0711 - } - - mntPointSymlinkExist := shared.PathExists(mountPointSymlink) - mntPointSymlinkTargetExist := shared.PathExists(mountPoint) - - var err error - if !mntPointSymlinkTargetExist { - err = os.MkdirAll(mountPoint, 0711) - if err != nil { - return err - } - } - - err = os.Chmod(mountPoint, mode) - if err != nil { - return err - } - - if !mntPointSymlinkExist { - err := os.Symlink(mountPoint, mountPointSymlink) - if err != nil { - return err - } - } - - return nil -} - func deleteContainerMountpoint(mountPoint string, mountPointSymlink string, storageTypeName string) error { if shared.PathExists(mountPointSymlink) { err := os.Remove(mountPointSymlink) @@ -698,27 +664,6 @@ func renameContainerMountpoint(oldMountPoint string, oldMountPointSymlink string return nil } -func createSnapshotMountpoint(snapshotMountpoint string, snapshotsSymlinkTarget string, snapshotsSymlink string) error { - snapshotMntPointExists := shared.PathExists(snapshotMountpoint) - mntPointSymlinkExist := shared.PathExists(snapshotsSymlink) - - if !snapshotMntPointExists { - err := os.MkdirAll(snapshotMountpoint, 0711) - if err != nil { - return err - } - } - - if !mntPointSymlinkExist { - err := os.Symlink(snapshotsSymlinkTarget, snapshotsSymlink) - if err != nil { - return err - } - } - - return nil -} - func deleteSnapshotMountpoint(snapshotMountpoint string, snapshotsSymlinkTarget string, snapshotsSymlink string) error { if shared.PathExists(snapshotMountpoint) { err := os.Remove(snapshotMountpoint) diff --git a/lxd/storage/storage.go b/lxd/storage/storage.go index f55c9f4ef4..b8d8d42b31 100644 --- a/lxd/storage/storage.go +++ b/lxd/storage/storage.go @@ -1,6 +1,8 @@ package storage import ( + "os" + "github.com/lxc/lxd/lxd/project" "github.com/lxc/lxd/shared" ) @@ -49,3 +51,61 @@ func GetStoragePoolVolumeMountPoint(poolName string, volumeName string) string { func GetStoragePoolVolumeSnapshotMountPoint(poolName string, snapshotName string) string { return shared.VarPath("storage-pools", poolName, "custom-snapshots", snapshotName) } + +// CreateContainerMountpoint creates the provided container mountpoint and symlink. +func CreateContainerMountpoint(mountPoint string, mountPointSymlink string, privileged bool) error { + var mode os.FileMode + if privileged { + mode = 0700 + } else { + mode = 0711 + } + + mntPointSymlinkExist := shared.PathExists(mountPointSymlink) + mntPointSymlinkTargetExist := shared.PathExists(mountPoint) + + var err error + if !mntPointSymlinkTargetExist { + err = os.MkdirAll(mountPoint, 0711) + if err != nil { + return err + } + } + + err = os.Chmod(mountPoint, mode) + if err != nil { + return err + } + + if !mntPointSymlinkExist { + err := os.Symlink(mountPoint, mountPointSymlink) + if err != nil { + return err + } + } + + return nil +} + +// CreateSnapshotMountpoint creates the provided container snapshot mountpoint +// and symlink. +func CreateSnapshotMountpoint(snapshotMountpoint string, snapshotsSymlinkTarget string, snapshotsSymlink string) error { + snapshotMntPointExists := shared.PathExists(snapshotMountpoint) + mntPointSymlinkExist := shared.PathExists(snapshotsSymlink) + + if !snapshotMntPointExists { + err := os.MkdirAll(snapshotMountpoint, 0711) + if err != nil { + return err + } + } + + if !mntPointSymlinkExist { + err := os.Symlink(snapshotsSymlinkTarget, snapshotsSymlink) + if err != nil { + return err + } + } + + return nil +} diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go index b9e133dd16..392fa71726 100644 --- a/lxd/storage_btrfs.go +++ b/lxd/storage_btrfs.go @@ -836,7 +836,7 @@ func (s *storageBtrfs) doContainerCreate(projectName, name string, privileged bo // Create the mountpoint for the container at: // ${LXD_DIR}/containers/<name> - err = createContainerMountpoint(containerSubvolumeName, shared.VarPath("containers", project.Prefix(projectName, name)), privileged) + err = driver.CreateContainerMountpoint(containerSubvolumeName, shared.VarPath("containers", project.Prefix(projectName, name)), privileged) if err != nil { return err } @@ -925,7 +925,7 @@ func (s *storageBtrfs) ContainerCreateFromImage(container container, fingerprint // Create the mountpoint for the container at: // ${LXD_DIR}/containers/<name> - err = createContainerMountpoint(containerSubvolumeName, container.Path(), container.IsPrivileged()) + err = driver.CreateContainerMountpoint(containerSubvolumeName, container.Path(), container.IsPrivileged()) if err != nil { return errors.Wrap(err, "Failed to create container mountpoint") } @@ -1006,7 +1006,7 @@ func (s *storageBtrfs) copyContainer(target container, source container) error { return err } - err = createContainerMountpoint(targetContainerSubvolumeName, target.Path(), target.IsPrivileged()) + err = driver.CreateContainerMountpoint(targetContainerSubvolumeName, target.Path(), target.IsPrivileged()) if err != nil { return err } @@ -1029,7 +1029,7 @@ func (s *storageBtrfs) copySnapshot(target container, source container) error { containersPath := driver.GetSnapshotMountPoint(target.Project(), s.pool.Name, targetParentName) snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(target.Project(), targetParentName)) snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(target.Project(), targetParentName)) - err := createSnapshotMountpoint(containersPath, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) + err := driver.CreateSnapshotMountpoint(containersPath, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) if err != nil { return err } @@ -1581,7 +1581,7 @@ func (s *storageBtrfs) ContainerSnapshotCreateEmpty(snapshotContainer container) snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(snapshotContainer.Project(), sourceName)) snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(snapshotContainer.Project(), sourceName)) if !shared.PathExists(snapshotMntPointSymlink) { - err := createContainerMountpoint(snapshotMntPointSymlinkTarget, snapshotMntPointSymlink, snapshotContainer.IsPrivileged()) + err := driver.CreateContainerMountpoint(snapshotMntPointSymlinkTarget, snapshotMntPointSymlink, snapshotContainer.IsPrivileged()) if err != nil { return err } @@ -1857,7 +1857,7 @@ func (s *storageBtrfs) doContainerBackupLoadOptimized(info backupInfo, data io.R snapshotMntPoint := driver.GetSnapshotMountPoint(info.Project, s.pool.Name, containerName) snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(info.Project, containerName)) snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(info.Project, containerName)) - err = createSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) + err = driver.CreateSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) if err != nil { feeder.Close() return err @@ -1900,7 +1900,7 @@ func (s *storageBtrfs) doContainerBackupLoadOptimized(info backupInfo, data io.R } // Create mountpoints - err = createContainerMountpoint(containerMntPoint, shared.VarPath("containers", project.Prefix(info.Project, info.Name)), info.Privileged) + err = driver.CreateContainerMountpoint(containerMntPoint, shared.VarPath("containers", project.Prefix(info.Project, info.Name)), info.Privileged) if err != nil { return err } @@ -2777,7 +2777,7 @@ func (s *storageBtrfs) MigrationSink(conn *websocket.Conn, op *operation, args M snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(args.Container.Project(), containerName)) snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(args.Container.Project(), containerName)) - err = createSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) + err = driver.CreateSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) if err != nil { return err } diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go index bd7bbbd4e8..09b7ffc367 100644 --- a/lxd/storage_ceph.go +++ b/lxd/storage_ceph.go @@ -878,7 +878,7 @@ func (s *storageCeph) ContainerCreateFromImage(container container, fingerprint // Create the mountpoint privileged := container.IsPrivileged() - err = createContainerMountpoint(containerPoolVolumeMntPoint, + err = driver.CreateContainerMountpoint(containerPoolVolumeMntPoint, containerPath, privileged) if err != nil { logger.Errorf(`Failed to create mountpoint "%s" for container "%s" for RBD storage volume: %s`, containerPoolVolumeMntPoint, containerName, err) @@ -1135,7 +1135,7 @@ func (s *storageCeph) ContainerCopy(target container, source container, target.Project(), s.pool.Name, targetContainerName) - err = createContainerMountpoint( + err = driver.CreateContainerMountpoint( targetContainerMountPoint, targetContainerPath, target.IsPrivileged()) @@ -1248,7 +1248,7 @@ func (s *storageCeph) ContainerCopy(target container, source container, "snapshots", project.Prefix(target.Project(), targetContainerName)) - err := createSnapshotMountpoint( + err := driver.CreateSnapshotMountpoint( containersPath, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) diff --git a/lxd/storage_ceph_migration.go b/lxd/storage_ceph_migration.go index 70e37a3e82..57e7839ef1 100644 --- a/lxd/storage_ceph_migration.go +++ b/lxd/storage_ceph_migration.go @@ -352,7 +352,7 @@ func (s *storageCeph) MigrationSink(conn *websocket.Conn, op *operation, args Mi } containerMntPoint := driver.GetContainerMountPoint(args.Container.Project(), s.pool.Name, containerName) - err = createContainerMountpoint( + err = driver.CreateContainerMountpoint( containerMntPoint, args.Container.Path(), args.Container.IsPrivileged()) diff --git a/lxd/storage_ceph_utils.go b/lxd/storage_ceph_utils.go index 104b151f4c..1a82a510f6 100644 --- a/lxd/storage_ceph_utils.go +++ b/lxd/storage_ceph_utils.go @@ -768,7 +768,7 @@ func (s *storageCeph) copyWithoutSnapshotsFull(target container, // Create mountpoint targetContainerMountPoint := driver.GetContainerMountPoint(target.Project(), s.pool.Name, target.Name()) - err = createContainerMountpoint(targetContainerMountPoint, target.Path(), target.IsPrivileged()) + err = driver.CreateContainerMountpoint(targetContainerMountPoint, target.Path(), target.IsPrivileged()) if err != nil { return err } @@ -849,7 +849,7 @@ func (s *storageCeph) copyWithoutSnapshotsSparse(target container, // Create mountpoint targetContainerMountPoint := driver.GetContainerMountPoint(target.Project(), s.pool.Name, target.Name()) - err = createContainerMountpoint(targetContainerMountPoint, target.Path(), target.IsPrivileged()) + err = driver.CreateContainerMountpoint(targetContainerMountPoint, target.Path(), target.IsPrivileged()) if err != nil { return err } @@ -1763,7 +1763,7 @@ func (s *storageCeph) doContainerCreate(projectName, name string, privileged boo containerPath := shared.VarPath("containers", project.Prefix(projectName, name)) containerMntPoint := driver.GetContainerMountPoint(projectName, s.pool.Name, name) - err = createContainerMountpoint(containerMntPoint, containerPath, privileged) + err = driver.CreateContainerMountpoint(containerMntPoint, containerPath, privileged) if err != nil { logger.Errorf(`Failed to create mountpoint "%s" for RBD storage volume for container "%s" on storage pool "%s": %s"`, containerMntPoint, name, s.pool.Name, err) return err @@ -1876,7 +1876,7 @@ func (s *storageCeph) doContainerSnapshotCreate(projectName, targetName string, sourceOnlyName, _, _ := shared.ContainerGetParentAndSnapshotName(sourceName) snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(projectName, sourceOnlyName)) snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(projectName, sourceOnlyName)) - err = createSnapshotMountpoint(targetContainerMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) + err = driver.CreateSnapshotMountpoint(targetContainerMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) if err != nil { logger.Errorf(`Failed to create mountpoint "%s", snapshot symlink target "%s", snapshot mountpoint symlink"%s" for RBD storage volume "%s" on storage pool "%s": %s`, targetContainerMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink, s.volume.Name, s.pool.Name, err) diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go index 21646c6111..3c38cedcfd 100644 --- a/lxd/storage_dir.go +++ b/lxd/storage_dir.go @@ -508,7 +508,7 @@ func (s *storageDir) ContainerCreate(container container) error { } containerMntPoint := driver.GetContainerMountPoint(container.Project(), s.pool.Name, container.Name()) - err = createContainerMountpoint(containerMntPoint, container.Path(), container.IsPrivileged()) + err = driver.CreateContainerMountpoint(containerMntPoint, container.Path(), container.IsPrivileged()) if err != nil { return err } @@ -552,7 +552,7 @@ func (s *storageDir) ContainerCreateFromImage(container container, imageFingerpr privileged := container.IsPrivileged() containerName := container.Name() containerMntPoint := driver.GetContainerMountPoint(container.Project(), s.pool.Name, containerName) - err = createContainerMountpoint(containerMntPoint, container.Path(), privileged) + err = driver.CreateContainerMountpoint(containerMntPoint, container.Path(), privileged) if err != nil { return errors.Wrap(err, "Create container mount point") } @@ -657,7 +657,7 @@ func (s *storageDir) copyContainer(target container, source container) error { } targetContainerMntPoint := driver.GetContainerMountPoint(target.Project(), targetPool, target.Name()) - err := createContainerMountpoint(targetContainerMntPoint, target.Path(), target.IsPrivileged()) + err := driver.CreateContainerMountpoint(targetContainerMntPoint, target.Path(), target.IsPrivileged()) if err != nil { return err } @@ -691,7 +691,7 @@ func (s *storageDir) copySnapshot(target container, targetPool string, source co containersPath := driver.GetSnapshotMountPoint(target.Project(), targetPool, targetParentName) snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", targetPool, "containers-snapshots", project.Prefix(target.Project(), targetParentName)) snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(target.Project(), targetParentName)) - err := createSnapshotMountpoint(containersPath, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) + err := driver.CreateSnapshotMountpoint(containersPath, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) if err != nil { return err } @@ -1027,7 +1027,7 @@ func (s *storageDir) ContainerSnapshotCreateEmpty(snapshotContainer container) e snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(snapshotContainer.Project(), sourceName)) snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(snapshotContainer.Project(), sourceName)) - err = createSnapshotMountpoint(targetContainerMntPoint, + err = driver.CreateSnapshotMountpoint(targetContainerMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) if err != nil { return err @@ -1222,7 +1222,7 @@ func (s *storageDir) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, ta // Create mountpoints containerMntPoint := driver.GetContainerMountPoint(info.Project, s.pool.Name, info.Name) - err = createContainerMountpoint(containerMntPoint, driver.ContainerPath(project.Prefix(info.Project, info.Name), false), info.Privileged) + err = driver.CreateContainerMountpoint(containerMntPoint, driver.ContainerPath(project.Prefix(info.Project, info.Name), false), info.Privileged) if err != nil { return errors.Wrap(err, "Create container mount point") } @@ -1248,7 +1248,7 @@ func (s *storageDir) ContainerBackupLoad(info backupInfo, data io.ReadSeeker, ta snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(info.Project, info.Name)) snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(info.Project, info.Name)) - err := createSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, + err := driver.CreateSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) if err != nil { return err diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go index 1a52f99e54..7f2b1f619d 100644 --- a/lxd/storage_lvm.go +++ b/lxd/storage_lvm.go @@ -965,7 +965,7 @@ func (s *storageLvm) ContainerCreate(container container) error { if err != nil { return err } - err = createSnapshotMountpoint(containerMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) + err = driver.CreateSnapshotMountpoint(containerMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) if err != nil { return err } @@ -976,7 +976,7 @@ func (s *storageLvm) ContainerCreate(container container) error { if err != nil { return err } - err = createContainerMountpoint(containerMntPoint, containerPath, container.IsPrivileged()) + err = driver.CreateContainerMountpoint(containerMntPoint, containerPath, container.IsPrivileged()) if err != nil { return err } @@ -1019,7 +1019,7 @@ func (s *storageLvm) ContainerCreateFromImage(container container, fingerprint s if err != nil { return errors.Wrapf(err, "Create container mount point directory at %s", containerMntPoint) } - err = createContainerMountpoint(containerMntPoint, containerPath, container.IsPrivileged()) + err = driver.CreateContainerMountpoint(containerMntPoint, containerPath, container.IsPrivileged()) if err != nil { return errors.Wrap(err, "Create container mount point") } @@ -1878,10 +1878,10 @@ func (s *storageLvm) doContainerBackupLoad(projectName, containerName string, pr cname, _, _ := shared.ContainerGetParentAndSnapshotName(containerName) snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(projectName, cname)) snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(projectName, cname)) - err = createSnapshotMountpoint(containerMntPoint, snapshotMntPointSymlinkTarget, + err = driver.CreateSnapshotMountpoint(containerMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) } else { - err = createContainerMountpoint(containerMntPoint, containerPath, privileged) + err = driver.CreateContainerMountpoint(containerMntPoint, containerPath, privileged) } if err != nil { return "", err diff --git a/lxd/storage_lvm_utils.go b/lxd/storage_lvm_utils.go index ff92174139..3c88a36888 100644 --- a/lxd/storage_lvm_utils.go +++ b/lxd/storage_lvm_utils.go @@ -288,10 +288,10 @@ func (s *storageLvm) createSnapshotContainer(snapshotContainer container, source sourceName, _, _ := shared.ContainerGetParentAndSnapshotName(sourceContainerName) snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(sourceContainer.Project(), sourceName)) snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(sourceContainer.Project(), sourceName)) - err = createSnapshotMountpoint(targetContainerMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) + err = driver.CreateSnapshotMountpoint(targetContainerMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) } else { targetContainerMntPoint = driver.GetContainerMountPoint(sourceContainer.Project(), targetPool, targetContainerName) - err = createContainerMountpoint(targetContainerMntPoint, targetContainerPath, snapshotContainer.IsPrivileged()) + err = driver.CreateContainerMountpoint(targetContainerMntPoint, targetContainerPath, snapshotContainer.IsPrivileged()) } if err != nil { return errors.Wrap(err, "Create mount point") @@ -351,7 +351,7 @@ func (s *storageLvm) copySnapshot(target container, source container, refresh bo containersPath := driver.GetSnapshotMountPoint(target.Project(), s.pool.Name, targetParentName) snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(target.Project(), targetParentName)) snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(target.Project(), targetParentName)) - err = createSnapshotMountpoint(containersPath, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) + err = driver.CreateSnapshotMountpoint(containersPath, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) if err != nil { return err } @@ -452,7 +452,7 @@ func (s *storageLvm) copyContainer(target container, source container, refresh b } targetContainerMntPoint := driver.GetContainerMountPoint(target.Project(), targetPool, target.Name()) - err = createContainerMountpoint(targetContainerMntPoint, target.Path(), target.IsPrivileged()) + err = driver.CreateContainerMountpoint(targetContainerMntPoint, target.Path(), target.IsPrivileged()) if err != nil { return err } diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go index 97b2901923..d467cfc364 100644 --- a/lxd/storage_zfs.go +++ b/lxd/storage_zfs.go @@ -900,7 +900,7 @@ func (s *storageZfs) ContainerCreateFromImage(container container, fingerprint s } privileged := container.IsPrivileged() - err = createContainerMountpoint(containerPoolVolumeMntPoint, containerPath, privileged) + err = driver.CreateContainerMountpoint(containerPoolVolumeMntPoint, containerPath, privileged) if err != nil { return err } @@ -988,7 +988,7 @@ func (s *storageZfs) copyWithoutSnapshotsSparse(target container, source contain defer s.ContainerUmount(target, targetContainerPath) } - err = createContainerMountpoint(targetContainerMountPoint, targetContainerPath, target.IsPrivileged()) + err = driver.CreateContainerMountpoint(targetContainerMountPoint, targetContainerPath, target.IsPrivileged()) if err != nil { return err } @@ -1119,7 +1119,7 @@ func (s *storageZfs) copyWithoutSnapshotFull(target container, source container) defer s.ContainerUmount(target, targetContainerMountPoint) } - err = createContainerMountpoint(targetContainerMountPoint, target.Path(), target.IsPrivileged()) + err = driver.CreateContainerMountpoint(targetContainerMountPoint, target.Path(), target.IsPrivileged()) if err != nil { return err } @@ -1134,7 +1134,7 @@ func (s *storageZfs) copyWithSnapshots(target container, source container, paren containersPath := driver.GetSnapshotMountPoint(target.Project(), s.pool.Name, targetParentName) snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(target.Project(), targetParentName)) snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(target.Project(), targetParentName)) - err := createSnapshotMountpoint(containersPath, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) + err := driver.CreateSnapshotMountpoint(containersPath, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) if err != nil { return err } @@ -1291,7 +1291,7 @@ func (s *storageZfs) ContainerCopy(target container, source container, container targetContainerName := target.Name() targetContainerPath := target.Path() targetContainerMountPoint := driver.GetContainerMountPoint(target.Project(), s.pool.Name, targetContainerName) - err = createContainerMountpoint(targetContainerMountPoint, targetContainerPath, target.IsPrivileged()) + err = driver.CreateContainerMountpoint(targetContainerMountPoint, targetContainerPath, target.IsPrivileged()) if err != nil { return err } @@ -2132,7 +2132,7 @@ func (s *storageZfs) ContainerBackupCreate(backup backup, source container) erro func (s *storageZfs) doContainerBackupLoadOptimized(info backupInfo, data io.ReadSeeker, tarArgs []string) error { containerName, _, _ := shared.ContainerGetParentAndSnapshotName(info.Name) containerMntPoint := driver.GetContainerMountPoint(info.Project, s.pool.Name, containerName) - err := createContainerMountpoint(containerMntPoint, driver.ContainerPath(info.Name, false), info.Privileged) + err := driver.CreateContainerMountpoint(containerMntPoint, driver.ContainerPath(info.Name, false), info.Privileged) if err != nil { return err } @@ -2192,7 +2192,7 @@ func (s *storageZfs) doContainerBackupLoadOptimized(info backupInfo, data io.Rea snapshotMntPoint := driver.GetSnapshotMountPoint(info.Project, s.pool.Name, fmt.Sprintf("%s/%s", containerName, snapshotOnlyName)) snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(info.Project, containerName)) snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(info.Project, containerName)) - err = createSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) + err = driver.CreateSnapshotMountpoint(snapshotMntPoint, snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) if err != nil { // can't use defer because it needs to run before the mount os.RemoveAll(unpackPath) diff --git a/lxd/storage_zfs_utils.go b/lxd/storage_zfs_utils.go index 5944c40407..4a2d87091f 100644 --- a/lxd/storage_zfs_utils.go +++ b/lxd/storage_zfs_utils.go @@ -670,7 +670,7 @@ func (s *storageZfs) doContainerMount(projectName, name string, privileged bool) // Since we're using mount() directly zfs will not automatically create // the mountpoint for us. So let's check and do it if needed. if !shared.PathExists(containerPoolVolumeMntPoint) { - err := createContainerMountpoint(containerPoolVolumeMntPoint, shared.VarPath(fs), privileged) + err := driver.CreateContainerMountpoint(containerPoolVolumeMntPoint, shared.VarPath(fs), privileged) if err != nil { return false, err } @@ -810,7 +810,7 @@ func (s *storageZfs) doContainerCreate(projectName, name string, privileged bool return err } - err = createContainerMountpoint(containerPoolVolumeMntPoint, containerPath, privileged) + err = driver.CreateContainerMountpoint(containerPoolVolumeMntPoint, containerPath, privileged) if err != nil { return err } From e81185b3a8c511152d68ede3e3adf1533454cf70 Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Mon, 19 Aug 2019 19:37:26 +0200 Subject: [PATCH 2/6] lxd: Move ChangeableStoragePoolProperties to storage Signed-off-by: Thomas Hipp <thomas.h...@canonical.com> --- lxd/storage/pools_config.go | 36 ++++++++++++++++++++++++++++++++++++ lxd/storage_btrfs.go | 2 +- lxd/storage_ceph.go | 4 ++-- lxd/storage_cephfs.go | 2 +- lxd/storage_dir.go | 2 +- lxd/storage_lvm.go | 2 +- lxd/storage_pools_config.go | 30 ------------------------------ lxd/storage_zfs.go | 2 +- 8 files changed, 43 insertions(+), 37 deletions(-) create mode 100644 lxd/storage/pools_config.go diff --git a/lxd/storage/pools_config.go b/lxd/storage/pools_config.go new file mode 100644 index 0000000000..6583acb168 --- /dev/null +++ b/lxd/storage/pools_config.go @@ -0,0 +1,36 @@ +package storage + +import "fmt" + +// ChangeableStoragePoolProperties is a map of storage backends and their +// changeable storage pool properties. +var ChangeableStoragePoolProperties = map[string][]string{ + "btrfs": { + "rsync.bwlimit", + "btrfs.mount_options"}, + + "ceph": { + "volume.block.filesystem", + "volume.block.mount_options", + "volume.size"}, + + "cephfs": { + "rsync.bwlimit"}, + + "dir": { + "rsync.bwlimit"}, + + "lvm": { + "lvm.thinpool_name", + "lvm.vg_name", + "volume.block.filesystem", + "volume.block.mount_options", + "volume.size"}, + + "zfs": { + "rsync_bwlimit", + "volume.zfs.remove_snapshots", + "volume.zfs.use_refquota", + "zfs.clone_copy"}, +} + diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go index 392fa71726..c5221f95b2 100644 --- a/lxd/storage_btrfs.go +++ b/lxd/storage_btrfs.go @@ -511,7 +511,7 @@ func (s *storageBtrfs) StoragePoolUpdate(writable *api.StoragePoolPut, changedConfig []string) error { logger.Infof(`Updating BTRFS storage pool "%s"`, s.pool.Name) - changeable := changeableStoragePoolProperties["btrfs"] + changeable := driver.ChangeableStoragePoolProperties["btrfs"] unchangeable := []string{} for _, change := range changedConfig { if !shared.StringInSlice(change, changeable) { diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go index 09b7ffc367..ddbc1d7f16 100644 --- a/lxd/storage_ceph.go +++ b/lxd/storage_ceph.go @@ -735,7 +735,7 @@ func (s *storageCeph) StoragePoolVolumeRename(newName string) error { func (s *storageCeph) StoragePoolUpdate(writable *api.StoragePoolPut, changedConfig []string) error { logger.Infof(`Updating CEPH storage pool "%s"`, s.pool.Name) - changeable := changeableStoragePoolProperties["ceph"] + changeable := driver.ChangeableStoragePoolProperties["ceph"] unchangeable := []string{} for _, change := range changedConfig { if !shared.StringInSlice(change, changeable) { @@ -744,7 +744,7 @@ func (s *storageCeph) StoragePoolUpdate(writable *api.StoragePoolPut, changedCon } if len(unchangeable) > 0 { - return updateStoragePoolError(unchangeable, "ceph") + return driver.UpdateStoragePoolError(unchangeable, "ceph") } // "rsync.bwlimit" requires no on-disk modifications. diff --git a/lxd/storage_cephfs.go b/lxd/storage_cephfs.go index 305de37498..b0457fd8cf 100644 --- a/lxd/storage_cephfs.go +++ b/lxd/storage_cephfs.go @@ -429,7 +429,7 @@ func (s *storageCephFs) StoragePoolUpdate(writable *api.StoragePoolPut, changedC logger.Infof(`Updating CEPHFS storage pool "%s"`, s.pool.Name) // Validate the properties - changeable := changeableStoragePoolProperties["cephfs"] + changeable := driver.ChangeableStoragePoolProperties["cephfs"] unchangeable := []string{} for _, change := range changedConfig { if !shared.StringInSlice(change, changeable) { diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go index 3c38cedcfd..e9f437aa1c 100644 --- a/lxd/storage_dir.go +++ b/lxd/storage_dir.go @@ -291,7 +291,7 @@ func (s *storageDir) StoragePoolUpdate(writable *api.StoragePoolPut, changedConf return err } - changeable := changeableStoragePoolProperties["dir"] + changeable := driver.ChangeableStoragePoolProperties["dir"] unchangeable := []string{} for _, change := range changedConfig { if !shared.StringInSlice(change, changeable) { diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go index 7f2b1f619d..ff97f8027f 100644 --- a/lxd/storage_lvm.go +++ b/lxd/storage_lvm.go @@ -688,7 +688,7 @@ func (s *storageLvm) GetContainerPoolInfo() (int64, string, string) { func (s *storageLvm) StoragePoolUpdate(writable *api.StoragePoolPut, changedConfig []string) error { logger.Infof(`Updating LVM storage pool "%s"`, s.pool.Name) - changeable := changeableStoragePoolProperties["lvm"] + changeable := driver.ChangeableStoragePoolProperties["lvm"] unchangeable := []string{} for _, change := range changedConfig { if !shared.StringInSlice(change, changeable) { diff --git a/lxd/storage_pools_config.go b/lxd/storage_pools_config.go index b44f0659df..1f06c0deee 100644 --- a/lxd/storage_pools_config.go +++ b/lxd/storage_pools_config.go @@ -16,36 +16,6 @@ func updateStoragePoolError(unchangeable []string, driverName string) error { `storage pools`, unchangeable, driverName) } -var changeableStoragePoolProperties = map[string][]string{ - "btrfs": { - "rsync.bwlimit", - "btrfs.mount_options"}, - - "ceph": { - "volume.block.filesystem", - "volume.block.mount_options", - "volume.size"}, - - "cephfs": { - "rsync.bwlimit"}, - - "dir": { - "rsync.bwlimit"}, - - "lvm": { - "lvm.thinpool_name", - "lvm.vg_name", - "volume.block.filesystem", - "volume.block.mount_options", - "volume.size"}, - - "zfs": { - "rsync_bwlimit", - "volume.zfs.remove_snapshots", - "volume.zfs.use_refquota", - "zfs.clone_copy"}, -} - var storagePoolConfigKeys = map[string]func(value string) error{ // valid drivers: btrfs // (Note, that we can't be smart in detecting mount options since a lot diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go index d467cfc364..0d98cf2977 100644 --- a/lxd/storage_zfs.go +++ b/lxd/storage_zfs.go @@ -629,7 +629,7 @@ func (s *storageZfs) GetContainerPoolInfo() (int64, string, string) { func (s *storageZfs) StoragePoolUpdate(writable *api.StoragePoolPut, changedConfig []string) error { logger.Infof(`Updating ZFS storage pool "%s"`, s.pool.Name) - changeable := changeableStoragePoolProperties["zfs"] + changeable := driver.ChangeableStoragePoolProperties["zfs"] unchangeable := []string{} for _, change := range changedConfig { if !shared.StringInSlice(change, changeable) { From cd1f65f6807644d87236bf47b09c7a09724c0c69 Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Thu, 29 Aug 2019 20:38:29 +0200 Subject: [PATCH 3/6] lxd: Move UpdateStoragePoolError to storage Signed-off-by: Thomas Hipp <thomas.h...@canonical.com> --- lxd/storage/pools_config.go | 6 ++++++ lxd/storage_btrfs.go | 2 +- lxd/storage_cephfs.go | 2 +- lxd/storage_dir.go | 2 +- lxd/storage_lvm.go | 2 +- lxd/storage_pools_config.go | 5 ----- lxd/storage_zfs.go | 2 +- 7 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lxd/storage/pools_config.go b/lxd/storage/pools_config.go index 6583acb168..a0c1684c53 100644 --- a/lxd/storage/pools_config.go +++ b/lxd/storage/pools_config.go @@ -34,3 +34,9 @@ var ChangeableStoragePoolProperties = map[string][]string{ "zfs.clone_copy"}, } +// UpdateStoragePoolError returns an error stating that the provided properties +// cannot be changed in the given backend. +func UpdateStoragePoolError(unchangeable []string, driverName string) error { + return fmt.Errorf("The %v properties cannot be changed for %q storage pools", + unchangeable, driverName) +} diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go index c5221f95b2..3c10da58fd 100644 --- a/lxd/storage_btrfs.go +++ b/lxd/storage_btrfs.go @@ -520,7 +520,7 @@ func (s *storageBtrfs) StoragePoolUpdate(writable *api.StoragePoolPut, } if len(unchangeable) > 0 { - return updateStoragePoolError(unchangeable, "btrfs") + return driver.UpdateStoragePoolError(unchangeable, "btrfs") } // "rsync.bwlimit" requires no on-disk modifications. diff --git a/lxd/storage_cephfs.go b/lxd/storage_cephfs.go index b0457fd8cf..3fe207b143 100644 --- a/lxd/storage_cephfs.go +++ b/lxd/storage_cephfs.go @@ -438,7 +438,7 @@ func (s *storageCephFs) StoragePoolUpdate(writable *api.StoragePoolPut, changedC } if len(unchangeable) > 0 { - return updateStoragePoolError(unchangeable, "cephfs") + return driver.UpdateStoragePoolError(unchangeable, "cephfs") } logger.Infof(`Updated CEPHFS storage pool "%s"`, s.pool.Name) diff --git a/lxd/storage_dir.go b/lxd/storage_dir.go index e9f437aa1c..736700304b 100644 --- a/lxd/storage_dir.go +++ b/lxd/storage_dir.go @@ -300,7 +300,7 @@ func (s *storageDir) StoragePoolUpdate(writable *api.StoragePoolPut, changedConf } if len(unchangeable) > 0 { - return updateStoragePoolError(unchangeable, "dir") + return driver.UpdateStoragePoolError(unchangeable, "dir") } // "rsync.bwlimit" requires no on-disk modifications. diff --git a/lxd/storage_lvm.go b/lxd/storage_lvm.go index ff97f8027f..ff8e06b703 100644 --- a/lxd/storage_lvm.go +++ b/lxd/storage_lvm.go @@ -697,7 +697,7 @@ func (s *storageLvm) StoragePoolUpdate(writable *api.StoragePoolPut, changedConf } if len(unchangeable) > 0 { - return updateStoragePoolError(unchangeable, "lvm") + return driver.UpdateStoragePoolError(unchangeable, "lvm") } // "volume.block.mount_options" requires no on-disk modifications. diff --git a/lxd/storage_pools_config.go b/lxd/storage_pools_config.go index 1f06c0deee..6ecd411ade 100644 --- a/lxd/storage_pools_config.go +++ b/lxd/storage_pools_config.go @@ -11,11 +11,6 @@ import ( "github.com/lxc/lxd/shared/units" ) -func updateStoragePoolError(unchangeable []string, driverName string) error { - return fmt.Errorf(`The %v properties cannot be changed for "%s" `+ - `storage pools`, unchangeable, driverName) -} - var storagePoolConfigKeys = map[string]func(value string) error{ // valid drivers: btrfs // (Note, that we can't be smart in detecting mount options since a lot diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go index 0d98cf2977..0087bd098c 100644 --- a/lxd/storage_zfs.go +++ b/lxd/storage_zfs.go @@ -638,7 +638,7 @@ func (s *storageZfs) StoragePoolUpdate(writable *api.StoragePoolPut, changedConf } if len(unchangeable) > 0 { - return updateStoragePoolError(unchangeable, "zfs") + return driver.UpdateStoragePoolError(unchangeable, "zfs") } // "rsync.bwlimit" requires no on-disk modifications. From b3b3bad101ee03bcc15f4768df5a54eb28d3c887 Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Fri, 30 Aug 2019 19:51:46 +0200 Subject: [PATCH 4/6] lxd: Move btrfs migration code The migration code needs to be moved to its own file since storage_btrfs.go will be removed in the future. The code however needs to stay in the main package (for now) since it depends on the container interface. Signed-off-by: Thomas Hipp <thomas.h...@canonical.com> --- lxd/storage_btrfs.go | 170 ------------------------------ lxd/storage_migration_btrfs.go | 185 +++++++++++++++++++++++++++++++++ 2 files changed, 185 insertions(+), 170 deletions(-) create mode 100644 lxd/storage_migration_btrfs.go diff --git a/lxd/storage_btrfs.go b/lxd/storage_btrfs.go index 3c10da58fd..f1cf4cd842 100644 --- a/lxd/storage_btrfs.go +++ b/lxd/storage_btrfs.go @@ -2430,170 +2430,6 @@ func btrfsSubVolumesGet(path string) ([]string, error) { return result, nil } -type btrfsMigrationSourceDriver struct { - container container - snapshots []container - btrfsSnapshotNames []string - btrfs *storageBtrfs - runningSnapName string - stoppedSnapName string -} - -func (s *btrfsMigrationSourceDriver) send(conn *websocket.Conn, btrfsPath string, btrfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) error { - args := []string{"send"} - if btrfsParent != "" { - args = append(args, "-p", btrfsParent) - } - args = append(args, btrfsPath) - - cmd := exec.Command("btrfs", args...) - - stdout, err := cmd.StdoutPipe() - if err != nil { - return err - } - - readPipe := io.ReadCloser(stdout) - if readWrapper != nil { - readPipe = readWrapper(stdout) - } - - stderr, err := cmd.StderrPipe() - if err != nil { - return err - } - - err = cmd.Start() - if err != nil { - return err - } - - <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024) - - output, err := ioutil.ReadAll(stderr) - if err != nil { - logger.Errorf("Problem reading btrfs send stderr: %s", err) - } - - err = cmd.Wait() - if err != nil { - logger.Errorf("Problem with btrfs send: %s", string(output)) - } - - return err -} - -func (s *btrfsMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn, op *operation, bwlimit string, containerOnly bool) error { - _, containerPool, _ := s.container.Storage().GetContainerPoolInfo() - containerName := s.container.Name() - containersPath := driver.GetContainerMountPoint("default", containerPool, "") - sourceName := containerName - - // Deal with sending a snapshot to create a container on another LXD - // instance. - if s.container.IsSnapshot() { - sourceName, _, _ := shared.ContainerGetParentAndSnapshotName(containerName) - snapshotsPath := driver.GetSnapshotMountPoint(s.container.Project(), containerPool, sourceName) - tmpContainerMntPoint, err := ioutil.TempDir(snapshotsPath, sourceName) - if err != nil { - return err - } - defer os.RemoveAll(tmpContainerMntPoint) - - err = os.Chmod(tmpContainerMntPoint, 0700) - if err != nil { - return err - } - - migrationSendSnapshot := fmt.Sprintf("%s/.migration-send", tmpContainerMntPoint) - snapshotMntPoint := driver.GetSnapshotMountPoint(s.container.Project(), containerPool, containerName) - err = s.btrfs.btrfsPoolVolumesSnapshot(snapshotMntPoint, migrationSendSnapshot, true, true) - if err != nil { - return err - } - defer btrfsSubVolumesDelete(migrationSendSnapshot) - - wrapper := StorageProgressReader(op, "fs_progress", containerName) - return s.send(conn, migrationSendSnapshot, "", wrapper) - } - - if !containerOnly { - for i, snap := range s.snapshots { - prev := "" - if i > 0 { - prev = driver.GetSnapshotMountPoint(snap.Project(), containerPool, s.snapshots[i-1].Name()) - } - - snapMntPoint := driver.GetSnapshotMountPoint(snap.Project(), containerPool, snap.Name()) - wrapper := StorageProgressReader(op, "fs_progress", snap.Name()) - if err := s.send(conn, snapMntPoint, prev, wrapper); err != nil { - return err - } - } - } - - tmpContainerMntPoint, err := ioutil.TempDir(containersPath, containerName) - if err != nil { - return err - } - defer os.RemoveAll(tmpContainerMntPoint) - - err = os.Chmod(tmpContainerMntPoint, 0700) - if err != nil { - return err - } - - migrationSendSnapshot := fmt.Sprintf("%s/.migration-send", tmpContainerMntPoint) - containerMntPoint := driver.GetContainerMountPoint(s.container.Project(), containerPool, sourceName) - err = s.btrfs.btrfsPoolVolumesSnapshot(containerMntPoint, migrationSendSnapshot, true, true) - if err != nil { - return err - } - defer btrfsSubVolumesDelete(migrationSendSnapshot) - - btrfsParent := "" - if len(s.btrfsSnapshotNames) > 0 { - btrfsParent = s.btrfsSnapshotNames[len(s.btrfsSnapshotNames)-1] - } - - wrapper := StorageProgressReader(op, "fs_progress", containerName) - return s.send(conn, migrationSendSnapshot, btrfsParent, wrapper) -} - -func (s *btrfsMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, bwlimit string) error { - tmpPath := driver.GetSnapshotMountPoint(s.container.Project(), s.btrfs.pool.Name, - fmt.Sprintf("%s/.migration-send", s.container.Name())) - err := os.MkdirAll(tmpPath, 0711) - if err != nil { - return err - } - - err = os.Chmod(tmpPath, 0700) - if err != nil { - return err - } - - s.stoppedSnapName = fmt.Sprintf("%s/.root", tmpPath) - parentName, _, _ := shared.ContainerGetParentAndSnapshotName(s.container.Name()) - containerMntPt := driver.GetContainerMountPoint(s.container.Project(), s.btrfs.pool.Name, parentName) - err = s.btrfs.btrfsPoolVolumesSnapshot(containerMntPt, s.stoppedSnapName, true, true) - if err != nil { - return err - } - - return s.send(conn, s.stoppedSnapName, s.runningSnapName, nil) -} - -func (s *btrfsMigrationSourceDriver) Cleanup() { - if s.stoppedSnapName != "" { - btrfsSubVolumesDelete(s.stoppedSnapName) - } - - if s.runningSnapName != "" { - btrfsSubVolumesDelete(s.runningSnapName) - } -} - func (s *storageBtrfs) MigrationType() migration.MigrationFSType { if s.s.OS.RunningInUserNS { return migration.MigrationFSType_RSYNC @@ -3121,12 +2957,6 @@ func (s *storageBtrfs) doCrossPoolVolumeCopy(sourcePool string, sourceName strin return nil } -func (s *btrfsMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, op *operation, bwlimit string, storage storage, volumeOnly bool) error { - msg := fmt.Sprintf("Function not implemented") - logger.Errorf(msg) - return fmt.Errorf(msg) -} - func (s *storageBtrfs) StorageMigrationSource(args MigrationSourceArgs) (MigrationStorageSourceDriver, error) { return rsyncStorageMigrationSource(args) } diff --git a/lxd/storage_migration_btrfs.go b/lxd/storage_migration_btrfs.go new file mode 100644 index 0000000000..2f58187467 --- /dev/null +++ b/lxd/storage_migration_btrfs.go @@ -0,0 +1,185 @@ +package main + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + + "github.com/gorilla/websocket" + + driver "github.com/lxc/lxd/lxd/storage" + "github.com/lxc/lxd/shared" + "github.com/lxc/lxd/shared/logger" +) + +type btrfsMigrationSourceDriver struct { + container container + snapshots []container + btrfsSnapshotNames []string + btrfs *storageBtrfs + runningSnapName string + stoppedSnapName string +} + +func (s *btrfsMigrationSourceDriver) send(conn *websocket.Conn, btrfsPath string, btrfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) error { + args := []string{"send"} + if btrfsParent != "" { + args = append(args, "-p", btrfsParent) + } + args = append(args, btrfsPath) + + cmd := exec.Command("btrfs", args...) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + + readPipe := io.ReadCloser(stdout) + if readWrapper != nil { + readPipe = readWrapper(stdout) + } + + stderr, err := cmd.StderrPipe() + if err != nil { + return err + } + + err = cmd.Start() + if err != nil { + return err + } + + <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024) + + output, err := ioutil.ReadAll(stderr) + if err != nil { + logger.Errorf("Problem reading btrfs send stderr: %s", err) + } + + err = cmd.Wait() + if err != nil { + logger.Errorf("Problem with btrfs send: %s", string(output)) + } + + return err +} + +func (s *btrfsMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn, op *operation, bwlimit string, containerOnly bool) error { + _, containerPool, _ := s.container.Storage().GetContainerPoolInfo() + containerName := s.container.Name() + containersPath := driver.GetContainerMountPoint("default", containerPool, "") + sourceName := containerName + + // Deal with sending a snapshot to create a container on another LXD + // instance. + if s.container.IsSnapshot() { + sourceName, _, _ := shared.ContainerGetParentAndSnapshotName(containerName) + snapshotsPath := driver.GetSnapshotMountPoint(s.container.Project(), containerPool, sourceName) + tmpContainerMntPoint, err := ioutil.TempDir(snapshotsPath, sourceName) + if err != nil { + return err + } + defer os.RemoveAll(tmpContainerMntPoint) + + err = os.Chmod(tmpContainerMntPoint, 0700) + if err != nil { + return err + } + + migrationSendSnapshot := fmt.Sprintf("%s/.migration-send", tmpContainerMntPoint) + snapshotMntPoint := driver.GetSnapshotMountPoint(s.container.Project(), containerPool, containerName) + err = s.btrfs.btrfsPoolVolumesSnapshot(snapshotMntPoint, migrationSendSnapshot, true, true) + if err != nil { + return err + } + defer btrfsSubVolumesDelete(migrationSendSnapshot) + + wrapper := StorageProgressReader(op, "fs_progress", containerName) + return s.send(conn, migrationSendSnapshot, "", wrapper) + } + + if !containerOnly { + for i, snap := range s.snapshots { + prev := "" + if i > 0 { + prev = driver.GetSnapshotMountPoint(snap.Project(), containerPool, s.snapshots[i-1].Name()) + } + + snapMntPoint := driver.GetSnapshotMountPoint(snap.Project(), containerPool, snap.Name()) + wrapper := StorageProgressReader(op, "fs_progress", snap.Name()) + if err := s.send(conn, snapMntPoint, prev, wrapper); err != nil { + return err + } + } + } + + tmpContainerMntPoint, err := ioutil.TempDir(containersPath, containerName) + if err != nil { + return err + } + defer os.RemoveAll(tmpContainerMntPoint) + + err = os.Chmod(tmpContainerMntPoint, 0700) + if err != nil { + return err + } + + migrationSendSnapshot := fmt.Sprintf("%s/.migration-send", tmpContainerMntPoint) + containerMntPoint := driver.GetContainerMountPoint(s.container.Project(), containerPool, sourceName) + err = s.btrfs.btrfsPoolVolumesSnapshot(containerMntPoint, migrationSendSnapshot, true, true) + if err != nil { + return err + } + defer btrfsSubVolumesDelete(migrationSendSnapshot) + + btrfsParent := "" + if len(s.btrfsSnapshotNames) > 0 { + btrfsParent = s.btrfsSnapshotNames[len(s.btrfsSnapshotNames)-1] + } + + wrapper := StorageProgressReader(op, "fs_progress", containerName) + return s.send(conn, migrationSendSnapshot, btrfsParent, wrapper) +} + +func (s *btrfsMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, bwlimit string) error { + tmpPath := driver.GetSnapshotMountPoint(s.container.Project(), s.btrfs.pool.Name, + fmt.Sprintf("%s/.migration-send", s.container.Name())) + err := os.MkdirAll(tmpPath, 0711) + if err != nil { + return err + } + + err = os.Chmod(tmpPath, 0700) + if err != nil { + return err + } + + s.stoppedSnapName = fmt.Sprintf("%s/.root", tmpPath) + parentName, _, _ := shared.ContainerGetParentAndSnapshotName(s.container.Name()) + containerMntPt := driver.GetContainerMountPoint(s.container.Project(), s.btrfs.pool.Name, parentName) + err = s.btrfs.btrfsPoolVolumesSnapshot(containerMntPt, s.stoppedSnapName, true, true) + if err != nil { + return err + } + + return s.send(conn, s.stoppedSnapName, s.runningSnapName, nil) +} + +func (s *btrfsMigrationSourceDriver) Cleanup() { + if s.stoppedSnapName != "" { + btrfsSubVolumesDelete(s.stoppedSnapName) + } + + if s.runningSnapName != "" { + btrfsSubVolumesDelete(s.runningSnapName) + } +} + +func (s *btrfsMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, op *operation, bwlimit string, storage storage, volumeOnly bool) error { + msg := fmt.Sprintf("Function not implemented") + logger.Errorf(msg) + return fmt.Errorf(msg) +} From 2264dbe92e20196f7d1713184a3e18083dd843dc Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Fri, 30 Aug 2019 20:04:04 +0200 Subject: [PATCH 5/6] lxd: Move zfs migration code The migration code needs to be moved to its own file since storage_zfs.go will be removed in the future. The code however needs to stay in the main package (for now) since it depends on the container interface. Signed-off-by: Thomas Hipp <thomas.h...@canonical.com> --- lxd/storage_migration_zfs.go | 146 +++++++++++++++++++++++++++++++++++ lxd/storage_zfs.go | 131 ------------------------------- 2 files changed, 146 insertions(+), 131 deletions(-) create mode 100644 lxd/storage_migration_zfs.go diff --git a/lxd/storage_migration_zfs.go b/lxd/storage_migration_zfs.go new file mode 100644 index 0000000000..fcfad12e5c --- /dev/null +++ b/lxd/storage_migration_zfs.go @@ -0,0 +1,146 @@ +package main + +import ( + "fmt" + "io" + "io/ioutil" + "os/exec" + + "github.com/gorilla/websocket" + "github.com/pborman/uuid" + + "github.com/lxc/lxd/lxd/project" + "github.com/lxc/lxd/shared" + "github.com/lxc/lxd/shared/logger" +) + +type zfsMigrationSourceDriver struct { + container container + snapshots []container + zfsSnapshotNames []string + zfs *storageZfs + runningSnapName string + stoppedSnapName string + zfsFeatures []string +} + +func (s *zfsMigrationSourceDriver) send(conn *websocket.Conn, zfsName string, zfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) error { + sourceParentName, _, _ := shared.ContainerGetParentAndSnapshotName(s.container.Name()) + poolName := s.zfs.getOnDiskPoolName() + args := []string{"send"} + + // Negotiated options + if s.zfsFeatures != nil && len(s.zfsFeatures) > 0 { + if shared.StringInSlice("compress", s.zfsFeatures) { + args = append(args, "-c") + args = append(args, "-L") + } + } + + args = append(args, []string{fmt.Sprintf("%s/containers/%s@%s", poolName, project.Prefix(s.container.Project(), sourceParentName), zfsName)}...) + if zfsParent != "" { + args = append(args, "-i", fmt.Sprintf("%s/containers/%s@%s", poolName, project.Prefix(s.container.Project(), s.container.Name()), zfsParent)) + } + + cmd := exec.Command("zfs", args...) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + + readPipe := io.ReadCloser(stdout) + if readWrapper != nil { + readPipe = readWrapper(stdout) + } + + stderr, err := cmd.StderrPipe() + if err != nil { + return err + } + + if err := cmd.Start(); err != nil { + return err + } + + <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024) + + output, err := ioutil.ReadAll(stderr) + if err != nil { + logger.Errorf("Problem reading zfs send stderr: %s", err) + } + + err = cmd.Wait() + if err != nil { + logger.Errorf("Problem with zfs send: %s", string(output)) + } + + return err +} + +func (s *zfsMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn, op *operation, bwlimit string, containerOnly bool) error { + if s.container.IsSnapshot() { + _, snapOnlyName, _ := shared.ContainerGetParentAndSnapshotName(s.container.Name()) + snapshotName := fmt.Sprintf("snapshot-%s", snapOnlyName) + wrapper := StorageProgressReader(op, "fs_progress", s.container.Name()) + return s.send(conn, snapshotName, "", wrapper) + } + + lastSnap := "" + if !containerOnly { + for i, snap := range s.zfsSnapshotNames { + prev := "" + if i > 0 { + prev = s.zfsSnapshotNames[i-1] + } + + lastSnap = snap + + wrapper := StorageProgressReader(op, "fs_progress", snap) + if err := s.send(conn, snap, prev, wrapper); err != nil { + return err + } + } + } + + s.runningSnapName = fmt.Sprintf("migration-send-%s", uuid.NewRandom().String()) + if err := zfsPoolVolumeSnapshotCreate(s.zfs.getOnDiskPoolName(), fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), s.container.Name())), s.runningSnapName); err != nil { + return err + } + + wrapper := StorageProgressReader(op, "fs_progress", s.container.Name()) + if err := s.send(conn, s.runningSnapName, lastSnap, wrapper); err != nil { + return err + } + + return nil +} + +func (s *zfsMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, bwlimit string) error { + s.stoppedSnapName = fmt.Sprintf("migration-send-%s", uuid.NewRandom().String()) + if err := zfsPoolVolumeSnapshotCreate(s.zfs.getOnDiskPoolName(), fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), s.container.Name())), s.stoppedSnapName); err != nil { + return err + } + + if err := s.send(conn, s.stoppedSnapName, s.runningSnapName, nil); err != nil { + return err + } + + return nil +} + +func (s *zfsMigrationSourceDriver) Cleanup() { + poolName := s.zfs.getOnDiskPoolName() + if s.stoppedSnapName != "" { + zfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), s.container.Name())), s.stoppedSnapName) + } + if s.runningSnapName != "" { + zfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), s.container.Name())), s.runningSnapName) + } +} + +func (s *zfsMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, op *operation, bwlimit string, storage storage, volumeOnly bool) error { + msg := fmt.Sprintf("Function not implemented") + logger.Errorf(msg) + return fmt.Errorf(msg) +} diff --git a/lxd/storage_zfs.go b/lxd/storage_zfs.go index 0087bd098c..93585d1ba6 100644 --- a/lxd/storage_zfs.go +++ b/lxd/storage_zfs.go @@ -2506,131 +2506,6 @@ func (s *storageZfs) ImageUmount(fingerprint string) (bool, error) { return true, nil } -type zfsMigrationSourceDriver struct { - container container - snapshots []container - zfsSnapshotNames []string - zfs *storageZfs - runningSnapName string - stoppedSnapName string - zfsFeatures []string -} - -func (s *zfsMigrationSourceDriver) send(conn *websocket.Conn, zfsName string, zfsParent string, readWrapper func(io.ReadCloser) io.ReadCloser) error { - sourceParentName, _, _ := shared.ContainerGetParentAndSnapshotName(s.container.Name()) - poolName := s.zfs.getOnDiskPoolName() - args := []string{"send"} - - // Negotiated options - if s.zfsFeatures != nil && len(s.zfsFeatures) > 0 { - if shared.StringInSlice("compress", s.zfsFeatures) { - args = append(args, "-c") - args = append(args, "-L") - } - } - - args = append(args, []string{fmt.Sprintf("%s/containers/%s@%s", poolName, project.Prefix(s.container.Project(), sourceParentName), zfsName)}...) - if zfsParent != "" { - args = append(args, "-i", fmt.Sprintf("%s/containers/%s@%s", poolName, project.Prefix(s.container.Project(), s.container.Name()), zfsParent)) - } - - cmd := exec.Command("zfs", args...) - - stdout, err := cmd.StdoutPipe() - if err != nil { - return err - } - - readPipe := io.ReadCloser(stdout) - if readWrapper != nil { - readPipe = readWrapper(stdout) - } - - stderr, err := cmd.StderrPipe() - if err != nil { - return err - } - - if err := cmd.Start(); err != nil { - return err - } - - <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024) - - output, err := ioutil.ReadAll(stderr) - if err != nil { - logger.Errorf("Problem reading zfs send stderr: %s", err) - } - - err = cmd.Wait() - if err != nil { - logger.Errorf("Problem with zfs send: %s", string(output)) - } - - return err -} - -func (s *zfsMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn, op *operation, bwlimit string, containerOnly bool) error { - if s.container.IsSnapshot() { - _, snapOnlyName, _ := shared.ContainerGetParentAndSnapshotName(s.container.Name()) - snapshotName := fmt.Sprintf("snapshot-%s", snapOnlyName) - wrapper := StorageProgressReader(op, "fs_progress", s.container.Name()) - return s.send(conn, snapshotName, "", wrapper) - } - - lastSnap := "" - if !containerOnly { - for i, snap := range s.zfsSnapshotNames { - prev := "" - if i > 0 { - prev = s.zfsSnapshotNames[i-1] - } - - lastSnap = snap - - wrapper := StorageProgressReader(op, "fs_progress", snap) - if err := s.send(conn, snap, prev, wrapper); err != nil { - return err - } - } - } - - s.runningSnapName = fmt.Sprintf("migration-send-%s", uuid.NewRandom().String()) - if err := zfsPoolVolumeSnapshotCreate(s.zfs.getOnDiskPoolName(), fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), s.container.Name())), s.runningSnapName); err != nil { - return err - } - - wrapper := StorageProgressReader(op, "fs_progress", s.container.Name()) - if err := s.send(conn, s.runningSnapName, lastSnap, wrapper); err != nil { - return err - } - - return nil -} - -func (s *zfsMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, bwlimit string) error { - s.stoppedSnapName = fmt.Sprintf("migration-send-%s", uuid.NewRandom().String()) - if err := zfsPoolVolumeSnapshotCreate(s.zfs.getOnDiskPoolName(), fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), s.container.Name())), s.stoppedSnapName); err != nil { - return err - } - - if err := s.send(conn, s.stoppedSnapName, s.runningSnapName, nil); err != nil { - return err - } - - return nil -} - -func (s *zfsMigrationSourceDriver) Cleanup() { - poolName := s.zfs.getOnDiskPoolName() - if s.stoppedSnapName != "" { - zfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), s.container.Name())), s.stoppedSnapName) - } - if s.runningSnapName != "" { - zfsPoolVolumeSnapshotDestroy(poolName, fmt.Sprintf("containers/%s", project.Prefix(s.container.Project(), s.container.Name())), s.runningSnapName) - } -} - func (s *storageZfs) MigrationType() migration.MigrationFSType { return migration.MigrationFSType_ZFS } @@ -3325,12 +3200,6 @@ func (s *storageZfs) StoragePoolVolumeCopy(source *api.StorageVolumeSource) erro return nil } -func (s *zfsMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, op *operation, bwlimit string, storage storage, volumeOnly bool) error { - msg := fmt.Sprintf("Function not implemented") - logger.Errorf(msg) - return fmt.Errorf(msg) -} - func (s *storageZfs) StorageMigrationSource(args MigrationSourceArgs) (MigrationStorageSourceDriver, error) { return rsyncStorageMigrationSource(args) } From 757e7f035825805fafba50fbd5cc68b94135ef04 Mon Sep 17 00:00:00 2001 From: Thomas Hipp <thomas.h...@canonical.com> Date: Fri, 30 Aug 2019 20:08:11 +0200 Subject: [PATCH 6/6] lxd: Move ceph migration code The migration code needs to be moved to its own file since storage_ceph.go will be removed in the future. The code however needs to stay in the main package (for now) since it depends on the container interface. Signed-off-by: Thomas Hipp <thomas.h...@canonical.com> --- lxd/storage_ceph.go | 256 +++++++++++++++++++ lxd/storage_ceph_migration.go | 366 ---------------------------- lxd/storage_ceph_migration_utils.go | 128 ---------- lxd/storage_migration_ceph.go | 225 +++++++++++++++++ 4 files changed, 481 insertions(+), 494 deletions(-) delete mode 100644 lxd/storage_ceph_migration.go delete mode 100644 lxd/storage_ceph_migration_utils.go create mode 100644 lxd/storage_migration_ceph.go diff --git a/lxd/storage_ceph.go b/lxd/storage_ceph.go index ddbc1d7f16..e28fb838b6 100644 --- a/lxd/storage_ceph.go +++ b/lxd/storage_ceph.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "os" + "os/exec" "strings" "github.com/gorilla/websocket" @@ -14,6 +15,7 @@ import ( "golang.org/x/sys/unix" "github.com/lxc/lxd/lxd/db" + "github.com/lxc/lxd/lxd/migration" "github.com/lxc/lxd/lxd/project" driver "github.com/lxc/lxd/lxd/storage" "github.com/lxc/lxd/shared" @@ -2804,3 +2806,257 @@ func (s *storageCeph) StoragePoolVolumeSnapshotRename(newName string) error { return s.s.Cluster.StoragePoolVolumeRename("default", s.volume.Name, fullSnapshotName, storagePoolVolumeTypeCustom, s.poolID) } + +func (s *storageCeph) MigrationType() migration.MigrationFSType { + return migration.MigrationFSType_RBD +} + +func (s *storageCeph) PreservesInodes() bool { + return false +} + +func (s *storageCeph) MigrationSource(args MigrationSourceArgs) (MigrationStorageSourceDriver, error) { + // If the container is a snapshot, let's just send that. We don't need + // to send anything else, because that's all the user asked for. + if args.Container.IsSnapshot() { + return &rbdMigrationSourceDriver{ + container: args.Container, + ceph: s, + }, nil + } + + driver := rbdMigrationSourceDriver{ + container: args.Container, + snapshots: []container{}, + rbdSnapshotNames: []string{}, + ceph: s, + } + + containerName := args.Container.Name() + if args.ContainerOnly { + logger.Debugf(`Only migrating the RBD storage volume for container "%s" on storage pool "%s`, containerName, s.pool.Name) + return &driver, nil + } + + // List all the snapshots in order of reverse creation. The idea here is + // that we send the oldest to newest snapshot, hopefully saving on xfer + // costs. Then, after all that, we send the container itself. + snapshots, err := cephRBDVolumeListSnapshots(s.ClusterName, + s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), + storagePoolVolumeTypeNameContainer, s.UserName) + if err != nil { + if err != db.ErrNoSuchObject { + logger.Errorf(`Failed to list snapshots for RBD storage volume "%s" on storage pool "%s": %s`, containerName, s.pool.Name, err) + return nil, err + } + } + logger.Debugf(`Retrieved snapshots "%v" for RBD storage volume "%s" on storage pool "%s"`, snapshots, containerName, s.pool.Name) + + for _, snap := range snapshots { + // In the case of e.g. multiple copies running at the same time, + // we will have potentially multiple migration-send snapshots. + // (Or in the case of the test suite, sometimes one will take + // too long to delete.) + if !strings.HasPrefix(snap, "snapshot_") { + continue + } + + lxdName := fmt.Sprintf("%s%s%s", containerName, shared.SnapshotDelimiter, snap[len("snapshot_"):]) + snapshot, err := containerLoadByProjectAndName(s.s, args.Container.Project(), lxdName) + if err != nil { + logger.Errorf(`Failed to load snapshot "%s" for RBD storage volume "%s" on storage pool "%s": %s`, lxdName, containerName, s.pool.Name, err) + return nil, err + } + + driver.snapshots = append(driver.snapshots, snapshot) + driver.rbdSnapshotNames = append(driver.rbdSnapshotNames, snap) + } + + return &driver, nil +} + +func (s *storageCeph) MigrationSink(conn *websocket.Conn, op *operation, args MigrationSinkArgs) error { + // Check that we received a valid root disk device with a pool property + // set. + parentStoragePool := "" + parentExpandedDevices := args.Container.ExpandedDevices() + parentLocalRootDiskDeviceKey, parentLocalRootDiskDevice, _ := shared.GetRootDiskDevice(parentExpandedDevices) + if parentLocalRootDiskDeviceKey != "" { + parentStoragePool = parentLocalRootDiskDevice["pool"] + } + + // A little neuroticism. + if parentStoragePool == "" { + return fmt.Errorf(`Detected that the container's root device ` + + `is missing the pool property during RBD migration`) + } + logger.Debugf(`Detected root disk device with pool property set to "%s" during RBD migration`, parentStoragePool) + + // create empty volume for container + // TODO: The cluster name can be different between LXD instances. Find + // out what to do in this case. Maybe I'm overthinking this and if the + // pool exists and we were able to initialize a new storage interface on + // the receiving LXD instance it also means that s.ClusterName has been + // set to the correct cluster name for that LXD instance. Yeah, I think + // that's actually correct. + containerName := args.Container.Name() + if !cephRBDVolumeExists(s.ClusterName, s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), storagePoolVolumeTypeNameContainer, s.UserName) { + err := cephRBDVolumeCreate(s.ClusterName, s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), storagePoolVolumeTypeNameContainer, "0", s.UserName) + if err != nil { + logger.Errorf(`Failed to create RBD storage volume "%s" for cluster "%s" in OSD pool "%s" on storage pool "%s": %s`, containerName, s.ClusterName, s.OSDPoolName, s.pool.Name, err) + return err + } + logger.Debugf(`Created RBD storage volume "%s" on storage pool "%s"`, containerName, s.pool.Name) + } + + if len(args.Snapshots) > 0 { + snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(args.Container.Project(), containerName)) + snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(args.Container.Project(), containerName)) + if !shared.PathExists(snapshotMntPointSymlink) { + err := os.Symlink(snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) + if err != nil { + return err + } + } + } + + // Now we're ready to receive the actual fs. + recvName := fmt.Sprintf("%s/container_%s", s.OSDPoolName, project.Prefix(args.Container.Project(), containerName)) + for _, snap := range args.Snapshots { + curSnapName := snap.GetName() + ctArgs := snapshotProtobufToContainerArgs(args.Container.Project(), containerName, snap) + + // Ensure that snapshot and parent container have the same + // storage pool in their local root disk device. If the root + // disk device for the snapshot comes from a profile on the new + // instance as well we don't need to do anything. + if ctArgs.Devices != nil { + snapLocalRootDiskDeviceKey, _, _ := shared.GetRootDiskDevice(ctArgs.Devices) + if snapLocalRootDiskDeviceKey != "" { + ctArgs.Devices[snapLocalRootDiskDeviceKey]["pool"] = parentStoragePool + } + } + _, err := containerCreateEmptySnapshot(args.Container.DaemonState(), ctArgs) + if err != nil { + logger.Errorf(`Failed to create empty RBD storage volume for container "%s" on storage pool "%s: %s`, containerName, s.OSDPoolName, err) + return err + } + logger.Debugf(`Created empty RBD storage volume for container "%s" on storage pool "%s`, containerName, s.OSDPoolName) + + wrapper := StorageProgressWriter(op, "fs_progress", curSnapName) + err = s.rbdRecv(conn, recvName, wrapper) + if err != nil { + logger.Errorf(`Failed to receive RBD storage volume "%s": %s`, curSnapName, err) + return err + } + logger.Debugf(`Received RBD storage volume "%s"`, curSnapName) + + snapshotMntPoint := driver.GetSnapshotMountPoint(args.Container.Project(), s.pool.Name, fmt.Sprintf("%s/%s", containerName, *snap.Name)) + if !shared.PathExists(snapshotMntPoint) { + err := os.MkdirAll(snapshotMntPoint, 0700) + if err != nil { + return err + } + } + } + + defer func() { + snaps, err := cephRBDVolumeListSnapshots(s.ClusterName, s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), storagePoolVolumeTypeNameContainer, s.UserName) + if err == nil { + for _, snap := range snaps { + snapOnlyName, _, _ := shared.ContainerGetParentAndSnapshotName(snap) + if !strings.HasPrefix(snapOnlyName, "migration-send") { + continue + } + + err := cephRBDSnapshotDelete(s.ClusterName, s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), storagePoolVolumeTypeNameContainer, snapOnlyName, s.UserName) + if err != nil { + logger.Warnf(`Failed to delete RBD container storage for snapshot "%s" of container "%s"`, snapOnlyName, containerName) + } + } + } + }() + + // receive the container itself + wrapper := StorageProgressWriter(op, "fs_progress", containerName) + err := s.rbdRecv(conn, recvName, wrapper) + if err != nil { + logger.Errorf(`Failed to receive RBD storage volume "%s": %s`, recvName, err) + return err + } + logger.Debugf(`Received RBD storage volume "%s"`, recvName) + + if args.Live { + err := s.rbdRecv(conn, recvName, wrapper) + if err != nil { + logger.Errorf(`Failed to receive RBD storage volume "%s": %s`, recvName, err) + return err + } + logger.Debugf(`Received RBD storage volume "%s"`, recvName) + } + + // Re-generate the UUID + err = s.cephRBDGenerateUUID(project.Prefix(args.Container.Project(), args.Container.Name()), storagePoolVolumeTypeNameContainer) + if err != nil { + return err + } + + containerMntPoint := driver.GetContainerMountPoint(args.Container.Project(), s.pool.Name, containerName) + err = driver.CreateContainerMountpoint( + containerMntPoint, + args.Container.Path(), + args.Container.IsPrivileged()) + if err != nil { + logger.Errorf(`Failed to create mountpoint "%s" for RBD storage volume for container "%s" on storage pool "%s": %s"`, containerMntPoint, containerName, s.pool.Name, err) + return err + } + logger.Debugf(`Created mountpoint "%s" for RBD storage volume for container "%s" on storage pool "%s""`, containerMntPoint, containerName, s.pool.Name) + + return nil +} +func (s *storageCeph) rbdRecv(conn *websocket.Conn, + volumeName string, + writeWrapper func(io.WriteCloser) io.WriteCloser) error { + args := []string{ + "import-diff", + "--cluster", s.ClusterName, + "-", + volumeName, + } + + cmd := exec.Command("rbd", args...) + + stdin, err := cmd.StdinPipe() + if err != nil { + return err + } + + stderr, err := cmd.StderrPipe() + if err != nil { + return err + } + + err = cmd.Start() + if err != nil { + return err + } + + writePipe := io.WriteCloser(stdin) + if writeWrapper != nil { + writePipe = writeWrapper(stdin) + } + + <-shared.WebsocketRecvStream(writePipe, conn) + + output, err := ioutil.ReadAll(stderr) + if err != nil { + logger.Debugf(`Failed to read stderr output from "rbd import-diff": %s`, err) + } + + err = cmd.Wait() + if err != nil { + logger.Errorf(`Failed to perform "rbd import-diff": %s`, string(output)) + } + + return err +} diff --git a/lxd/storage_ceph_migration.go b/lxd/storage_ceph_migration.go deleted file mode 100644 index 57e7839ef1..0000000000 --- a/lxd/storage_ceph_migration.go +++ /dev/null @@ -1,366 +0,0 @@ -package main - -import ( - "fmt" - "os" - "strings" - - "github.com/gorilla/websocket" - "github.com/pborman/uuid" - - "github.com/lxc/lxd/lxd/db" - "github.com/lxc/lxd/lxd/migration" - "github.com/lxc/lxd/lxd/project" - driver "github.com/lxc/lxd/lxd/storage" - "github.com/lxc/lxd/shared" - "github.com/lxc/lxd/shared/logger" -) - -type rbdMigrationSourceDriver struct { - container container - snapshots []container - rbdSnapshotNames []string - ceph *storageCeph - runningSnapName string - stoppedSnapName string -} - -func (s *rbdMigrationSourceDriver) Snapshots() []container { - return s.snapshots -} - -func (s *rbdMigrationSourceDriver) Cleanup() { - containerName := s.container.Name() - - if s.stoppedSnapName != "" { - err := cephRBDSnapshotDelete(s.ceph.ClusterName, s.ceph.OSDPoolName, - project.Prefix(s.container.Project(), containerName), storagePoolVolumeTypeNameContainer, - s.stoppedSnapName, s.ceph.UserName) - if err != nil { - logger.Warnf(`Failed to delete RBD snapshot "%s" of container "%s"`, s.stoppedSnapName, containerName) - } - } - - if s.runningSnapName != "" { - err := cephRBDSnapshotDelete(s.ceph.ClusterName, s.ceph.OSDPoolName, - project.Prefix(s.container.Project(), containerName), storagePoolVolumeTypeNameContainer, - s.runningSnapName, s.ceph.UserName) - if err != nil { - logger.Warnf(`Failed to delete RBD snapshot "%s" of container "%s"`, s.runningSnapName, containerName) - } - } -} - -func (s *rbdMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, bwlimit string) error { - containerName := s.container.Name() - s.stoppedSnapName = fmt.Sprintf("migration-send-%s", uuid.NewRandom().String()) - err := cephRBDSnapshotCreate(s.ceph.ClusterName, s.ceph.OSDPoolName, - project.Prefix(s.container.Project(), containerName), storagePoolVolumeTypeNameContainer, - s.stoppedSnapName, s.ceph.UserName) - if err != nil { - logger.Errorf(`Failed to create snapshot "%s" for RBD storage volume for image "%s" on storage pool "%s": %s`, s.stoppedSnapName, containerName, s.ceph.pool.Name, err) - return err - } - - cur := fmt.Sprintf("%s/container_%s@%s", s.ceph.OSDPoolName, - project.Prefix(s.container.Project(), containerName), s.stoppedSnapName) - err = s.rbdSend(conn, cur, s.runningSnapName, nil) - if err != nil { - logger.Errorf(`Failed to send exported diff of RBD storage volume "%s" from snapshot "%s": %s`, cur, s.runningSnapName, err) - return err - } - logger.Debugf(`Sent exported diff of RBD storage volume "%s" from snapshot "%s"`, cur, s.stoppedSnapName) - - return nil -} - -func (s *rbdMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn, - op *operation, bwlimit string, containerOnly bool) error { - containerName := s.container.Name() - if s.container.IsSnapshot() { - // ContainerSnapshotStart() will create the clone that is - // referenced by sendName here. - containerOnlyName, snapOnlyName, _ := shared.ContainerGetParentAndSnapshotName(containerName) - sendName := fmt.Sprintf( - "%s/snapshots_%s_%s_start_clone", - s.ceph.OSDPoolName, - containerOnlyName, - snapOnlyName) - wrapper := StorageProgressReader(op, "fs_progress", containerName) - - err := s.rbdSend(conn, sendName, "", wrapper) - if err != nil { - logger.Errorf(`Failed to send RBD storage volume "%s": %s`, sendName, err) - return err - } - logger.Debugf(`Sent RBD storage volume "%s"`, sendName) - - return nil - } - - lastSnap := "" - if !containerOnly { - for i, snap := range s.rbdSnapshotNames { - prev := "" - if i > 0 { - prev = s.rbdSnapshotNames[i-1] - } - - lastSnap = snap - - sendSnapName := fmt.Sprintf( - "%s/container_%s@%s", - s.ceph.OSDPoolName, - project.Prefix(s.container.Project(), containerName), - snap) - - wrapper := StorageProgressReader(op, "fs_progress", snap) - - err := s.rbdSend( - conn, - sendSnapName, - prev, - wrapper) - if err != nil { - logger.Errorf(`Failed to send exported diff of RBD storage volume "%s" from snapshot "%s": %s`, sendSnapName, prev, err) - return err - } - logger.Debugf(`Sent exported diff of RBD storage volume "%s" from snapshot "%s"`, sendSnapName, prev) - } - } - - s.runningSnapName = fmt.Sprintf("migration-send-%s", uuid.NewRandom().String()) - err := cephRBDSnapshotCreate(s.ceph.ClusterName, s.ceph.OSDPoolName, - project.Prefix(s.container.Project(), containerName), storagePoolVolumeTypeNameContainer, - s.runningSnapName, s.ceph.UserName) - if err != nil { - logger.Errorf(`Failed to create snapshot "%s" for RBD storage volume for image "%s" on storage pool "%s": %s`, s.runningSnapName, containerName, s.ceph.pool.Name, err) - return err - } - - cur := fmt.Sprintf("%s/container_%s@%s", s.ceph.OSDPoolName, - project.Prefix(s.container.Project(), containerName), s.runningSnapName) - wrapper := StorageProgressReader(op, "fs_progress", containerName) - err = s.rbdSend(conn, cur, lastSnap, wrapper) - if err != nil { - logger.Errorf(`Failed to send exported diff of RBD storage volume "%s" from snapshot "%s": %s`, s.runningSnapName, lastSnap, err) - return err - } - logger.Debugf(`Sent exported diff of RBD storage volume "%s" from snapshot "%s"`, s.runningSnapName, lastSnap) - - return nil -} - -func (s *rbdMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, op *operation, bwlimit string, storage storage, volumeOnly bool) error { - msg := fmt.Sprintf("Function not implemented") - logger.Errorf(msg) - return fmt.Errorf(msg) -} - -func (s *storageCeph) MigrationType() migration.MigrationFSType { - return migration.MigrationFSType_RBD -} - -func (s *storageCeph) PreservesInodes() bool { - return false -} - -func (s *storageCeph) MigrationSource(args MigrationSourceArgs) (MigrationStorageSourceDriver, error) { - // If the container is a snapshot, let's just send that. We don't need - // to send anything else, because that's all the user asked for. - if args.Container.IsSnapshot() { - return &rbdMigrationSourceDriver{ - container: args.Container, - ceph: s, - }, nil - } - - driver := rbdMigrationSourceDriver{ - container: args.Container, - snapshots: []container{}, - rbdSnapshotNames: []string{}, - ceph: s, - } - - containerName := args.Container.Name() - if args.ContainerOnly { - logger.Debugf(`Only migrating the RBD storage volume for container "%s" on storage pool "%s`, containerName, s.pool.Name) - return &driver, nil - } - - // List all the snapshots in order of reverse creation. The idea here is - // that we send the oldest to newest snapshot, hopefully saving on xfer - // costs. Then, after all that, we send the container itself. - snapshots, err := cephRBDVolumeListSnapshots(s.ClusterName, - s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), - storagePoolVolumeTypeNameContainer, s.UserName) - if err != nil { - if err != db.ErrNoSuchObject { - logger.Errorf(`Failed to list snapshots for RBD storage volume "%s" on storage pool "%s": %s`, containerName, s.pool.Name, err) - return nil, err - } - } - logger.Debugf(`Retrieved snapshots "%v" for RBD storage volume "%s" on storage pool "%s"`, snapshots, containerName, s.pool.Name) - - for _, snap := range snapshots { - // In the case of e.g. multiple copies running at the same time, - // we will have potentially multiple migration-send snapshots. - // (Or in the case of the test suite, sometimes one will take - // too long to delete.) - if !strings.HasPrefix(snap, "snapshot_") { - continue - } - - lxdName := fmt.Sprintf("%s%s%s", containerName, shared.SnapshotDelimiter, snap[len("snapshot_"):]) - snapshot, err := containerLoadByProjectAndName(s.s, args.Container.Project(), lxdName) - if err != nil { - logger.Errorf(`Failed to load snapshot "%s" for RBD storage volume "%s" on storage pool "%s": %s`, lxdName, containerName, s.pool.Name, err) - return nil, err - } - - driver.snapshots = append(driver.snapshots, snapshot) - driver.rbdSnapshotNames = append(driver.rbdSnapshotNames, snap) - } - - return &driver, nil -} - -func (s *storageCeph) MigrationSink(conn *websocket.Conn, op *operation, args MigrationSinkArgs) error { - // Check that we received a valid root disk device with a pool property - // set. - parentStoragePool := "" - parentExpandedDevices := args.Container.ExpandedDevices() - parentLocalRootDiskDeviceKey, parentLocalRootDiskDevice, _ := shared.GetRootDiskDevice(parentExpandedDevices) - if parentLocalRootDiskDeviceKey != "" { - parentStoragePool = parentLocalRootDiskDevice["pool"] - } - - // A little neuroticism. - if parentStoragePool == "" { - return fmt.Errorf(`Detected that the container's root device ` + - `is missing the pool property during RBD migration`) - } - logger.Debugf(`Detected root disk device with pool property set to "%s" during RBD migration`, parentStoragePool) - - // create empty volume for container - // TODO: The cluster name can be different between LXD instances. Find - // out what to do in this case. Maybe I'm overthinking this and if the - // pool exists and we were able to initialize a new storage interface on - // the receiving LXD instance it also means that s.ClusterName has been - // set to the correct cluster name for that LXD instance. Yeah, I think - // that's actually correct. - containerName := args.Container.Name() - if !cephRBDVolumeExists(s.ClusterName, s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), storagePoolVolumeTypeNameContainer, s.UserName) { - err := cephRBDVolumeCreate(s.ClusterName, s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), storagePoolVolumeTypeNameContainer, "0", s.UserName) - if err != nil { - logger.Errorf(`Failed to create RBD storage volume "%s" for cluster "%s" in OSD pool "%s" on storage pool "%s": %s`, containerName, s.ClusterName, s.OSDPoolName, s.pool.Name, err) - return err - } - logger.Debugf(`Created RBD storage volume "%s" on storage pool "%s"`, containerName, s.pool.Name) - } - - if len(args.Snapshots) > 0 { - snapshotMntPointSymlinkTarget := shared.VarPath("storage-pools", s.pool.Name, "containers-snapshots", project.Prefix(args.Container.Project(), containerName)) - snapshotMntPointSymlink := shared.VarPath("snapshots", project.Prefix(args.Container.Project(), containerName)) - if !shared.PathExists(snapshotMntPointSymlink) { - err := os.Symlink(snapshotMntPointSymlinkTarget, snapshotMntPointSymlink) - if err != nil { - return err - } - } - } - - // Now we're ready to receive the actual fs. - recvName := fmt.Sprintf("%s/container_%s", s.OSDPoolName, project.Prefix(args.Container.Project(), containerName)) - for _, snap := range args.Snapshots { - curSnapName := snap.GetName() - ctArgs := snapshotProtobufToContainerArgs(args.Container.Project(), containerName, snap) - - // Ensure that snapshot and parent container have the same - // storage pool in their local root disk device. If the root - // disk device for the snapshot comes from a profile on the new - // instance as well we don't need to do anything. - if ctArgs.Devices != nil { - snapLocalRootDiskDeviceKey, _, _ := shared.GetRootDiskDevice(ctArgs.Devices) - if snapLocalRootDiskDeviceKey != "" { - ctArgs.Devices[snapLocalRootDiskDeviceKey]["pool"] = parentStoragePool - } - } - _, err := containerCreateEmptySnapshot(args.Container.DaemonState(), ctArgs) - if err != nil { - logger.Errorf(`Failed to create empty RBD storage volume for container "%s" on storage pool "%s: %s`, containerName, s.OSDPoolName, err) - return err - } - logger.Debugf(`Created empty RBD storage volume for container "%s" on storage pool "%s`, containerName, s.OSDPoolName) - - wrapper := StorageProgressWriter(op, "fs_progress", curSnapName) - err = s.rbdRecv(conn, recvName, wrapper) - if err != nil { - logger.Errorf(`Failed to receive RBD storage volume "%s": %s`, curSnapName, err) - return err - } - logger.Debugf(`Received RBD storage volume "%s"`, curSnapName) - - snapshotMntPoint := driver.GetSnapshotMountPoint(args.Container.Project(), s.pool.Name, fmt.Sprintf("%s/%s", containerName, *snap.Name)) - if !shared.PathExists(snapshotMntPoint) { - err := os.MkdirAll(snapshotMntPoint, 0700) - if err != nil { - return err - } - } - } - - defer func() { - snaps, err := cephRBDVolumeListSnapshots(s.ClusterName, s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), storagePoolVolumeTypeNameContainer, s.UserName) - if err == nil { - for _, snap := range snaps { - snapOnlyName, _, _ := shared.ContainerGetParentAndSnapshotName(snap) - if !strings.HasPrefix(snapOnlyName, "migration-send") { - continue - } - - err := cephRBDSnapshotDelete(s.ClusterName, s.OSDPoolName, project.Prefix(args.Container.Project(), containerName), storagePoolVolumeTypeNameContainer, snapOnlyName, s.UserName) - if err != nil { - logger.Warnf(`Failed to delete RBD container storage for snapshot "%s" of container "%s"`, snapOnlyName, containerName) - } - } - } - }() - - // receive the container itself - wrapper := StorageProgressWriter(op, "fs_progress", containerName) - err := s.rbdRecv(conn, recvName, wrapper) - if err != nil { - logger.Errorf(`Failed to receive RBD storage volume "%s": %s`, recvName, err) - return err - } - logger.Debugf(`Received RBD storage volume "%s"`, recvName) - - if args.Live { - err := s.rbdRecv(conn, recvName, wrapper) - if err != nil { - logger.Errorf(`Failed to receive RBD storage volume "%s": %s`, recvName, err) - return err - } - logger.Debugf(`Received RBD storage volume "%s"`, recvName) - } - - // Re-generate the UUID - err = s.cephRBDGenerateUUID(project.Prefix(args.Container.Project(), args.Container.Name()), storagePoolVolumeTypeNameContainer) - if err != nil { - return err - } - - containerMntPoint := driver.GetContainerMountPoint(args.Container.Project(), s.pool.Name, containerName) - err = driver.CreateContainerMountpoint( - containerMntPoint, - args.Container.Path(), - args.Container.IsPrivileged()) - if err != nil { - logger.Errorf(`Failed to create mountpoint "%s" for RBD storage volume for container "%s" on storage pool "%s": %s"`, containerMntPoint, containerName, s.pool.Name, err) - return err - } - logger.Debugf(`Created mountpoint "%s" for RBD storage volume for container "%s" on storage pool "%s""`, containerMntPoint, containerName, s.pool.Name) - - return nil -} diff --git a/lxd/storage_ceph_migration_utils.go b/lxd/storage_ceph_migration_utils.go deleted file mode 100644 index 77caee4d8d..0000000000 --- a/lxd/storage_ceph_migration_utils.go +++ /dev/null @@ -1,128 +0,0 @@ -package main - -import ( - "io" - "io/ioutil" - "os/exec" - - "github.com/gorilla/websocket" - - "github.com/lxc/lxd/shared" - "github.com/lxc/lxd/shared/logger" -) - -// Let's say we want to send the a container "a" including snapshots "snap0" and -// "snap1" on storage pool "pool1" from LXD "l1" to LXD "l2" on storage pool -// "pool2": -// -// The pool layout on "l1" would be: -// pool1/container_a -// pool1/container_a@snapshot_snap0 -// pool1/container_a@snapshot_snap1 -// -// Then we need to send: -// rbd export-diff pool1/container_a@snapshot_snap0 - | rbd import-diff - pool2/container_a -// (Note that pool2/container_a must have been created by the receiving LXD -// instance before.) -// rbd export-diff pool1/container_a@snapshot_snap1 --from-snap snapshot_snap0 - | rbd import-diff - pool2/container_a -// rbd export-diff pool1/container_a --from-snap snapshot_snap1 - | rbd import-diff - pool2/container_a -func (s *rbdMigrationSourceDriver) rbdSend(conn *websocket.Conn, - volumeName string, - volumeParentName string, - readWrapper func(io.ReadCloser) io.ReadCloser) error { - args := []string{ - "export-diff", - "--cluster", s.ceph.ClusterName, - volumeName, - } - - if volumeParentName != "" { - args = append(args, "--from-snap", volumeParentName) - } - - // redirect output to stdout - args = append(args, "-") - - cmd := exec.Command("rbd", args...) - - stdout, err := cmd.StdoutPipe() - if err != nil { - return err - } - - readPipe := io.ReadCloser(stdout) - if readWrapper != nil { - readPipe = readWrapper(stdout) - } - - stderr, err := cmd.StderrPipe() - if err != nil { - return err - } - - err = cmd.Start() - if err != nil { - return err - } - - <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024) - - output, err := ioutil.ReadAll(stderr) - if err != nil { - logger.Debugf(`Failed to read stderr output from "rbd export-diff": %s`, err) - } - - err = cmd.Wait() - if err != nil { - logger.Errorf(`Failed to perform "rbd export-diff": %s`, string(output)) - } - - return err -} - -func (s *storageCeph) rbdRecv(conn *websocket.Conn, - volumeName string, - writeWrapper func(io.WriteCloser) io.WriteCloser) error { - args := []string{ - "import-diff", - "--cluster", s.ClusterName, - "-", - volumeName, - } - - cmd := exec.Command("rbd", args...) - - stdin, err := cmd.StdinPipe() - if err != nil { - return err - } - - stderr, err := cmd.StderrPipe() - if err != nil { - return err - } - - err = cmd.Start() - if err != nil { - return err - } - - writePipe := io.WriteCloser(stdin) - if writeWrapper != nil { - writePipe = writeWrapper(stdin) - } - - <-shared.WebsocketRecvStream(writePipe, conn) - - output, err := ioutil.ReadAll(stderr) - if err != nil { - logger.Debugf(`Failed to read stderr output from "rbd import-diff": %s`, err) - } - - err = cmd.Wait() - if err != nil { - logger.Errorf(`Failed to perform "rbd import-diff": %s`, string(output)) - } - - return err -} diff --git a/lxd/storage_migration_ceph.go b/lxd/storage_migration_ceph.go new file mode 100644 index 0000000000..83cc7cfc0c --- /dev/null +++ b/lxd/storage_migration_ceph.go @@ -0,0 +1,225 @@ +package main + +import ( + "fmt" + "io" + "io/ioutil" + "os/exec" + + "github.com/gorilla/websocket" + "github.com/pborman/uuid" + + "github.com/lxc/lxd/lxd/project" + "github.com/lxc/lxd/shared" + "github.com/lxc/lxd/shared/logger" +) + +type rbdMigrationSourceDriver struct { + container container + snapshots []container + rbdSnapshotNames []string + ceph *storageCeph + runningSnapName string + stoppedSnapName string +} + +func (s *rbdMigrationSourceDriver) Snapshots() []container { + return s.snapshots +} + +func (s *rbdMigrationSourceDriver) Cleanup() { + containerName := s.container.Name() + + if s.stoppedSnapName != "" { + err := cephRBDSnapshotDelete(s.ceph.ClusterName, s.ceph.OSDPoolName, + project.Prefix(s.container.Project(), containerName), storagePoolVolumeTypeNameContainer, + s.stoppedSnapName, s.ceph.UserName) + if err != nil { + logger.Warnf(`Failed to delete RBD snapshot "%s" of container "%s"`, s.stoppedSnapName, containerName) + } + } + + if s.runningSnapName != "" { + err := cephRBDSnapshotDelete(s.ceph.ClusterName, s.ceph.OSDPoolName, + project.Prefix(s.container.Project(), containerName), storagePoolVolumeTypeNameContainer, + s.runningSnapName, s.ceph.UserName) + if err != nil { + logger.Warnf(`Failed to delete RBD snapshot "%s" of container "%s"`, s.runningSnapName, containerName) + } + } +} + +func (s *rbdMigrationSourceDriver) SendAfterCheckpoint(conn *websocket.Conn, bwlimit string) error { + containerName := s.container.Name() + s.stoppedSnapName = fmt.Sprintf("migration-send-%s", uuid.NewRandom().String()) + err := cephRBDSnapshotCreate(s.ceph.ClusterName, s.ceph.OSDPoolName, + project.Prefix(s.container.Project(), containerName), storagePoolVolumeTypeNameContainer, + s.stoppedSnapName, s.ceph.UserName) + if err != nil { + logger.Errorf(`Failed to create snapshot "%s" for RBD storage volume for image "%s" on storage pool "%s": %s`, s.stoppedSnapName, containerName, s.ceph.pool.Name, err) + return err + } + + cur := fmt.Sprintf("%s/container_%s@%s", s.ceph.OSDPoolName, + project.Prefix(s.container.Project(), containerName), s.stoppedSnapName) + err = s.rbdSend(conn, cur, s.runningSnapName, nil) + if err != nil { + logger.Errorf(`Failed to send exported diff of RBD storage volume "%s" from snapshot "%s": %s`, cur, s.runningSnapName, err) + return err + } + logger.Debugf(`Sent exported diff of RBD storage volume "%s" from snapshot "%s"`, cur, s.stoppedSnapName) + + return nil +} + +func (s *rbdMigrationSourceDriver) SendWhileRunning(conn *websocket.Conn, + op *operation, bwlimit string, containerOnly bool) error { + containerName := s.container.Name() + if s.container.IsSnapshot() { + // ContainerSnapshotStart() will create the clone that is + // referenced by sendName here. + containerOnlyName, snapOnlyName, _ := shared.ContainerGetParentAndSnapshotName(containerName) + sendName := fmt.Sprintf( + "%s/snapshots_%s_%s_start_clone", + s.ceph.OSDPoolName, + containerOnlyName, + snapOnlyName) + wrapper := StorageProgressReader(op, "fs_progress", containerName) + + err := s.rbdSend(conn, sendName, "", wrapper) + if err != nil { + logger.Errorf(`Failed to send RBD storage volume "%s": %s`, sendName, err) + return err + } + logger.Debugf(`Sent RBD storage volume "%s"`, sendName) + + return nil + } + + lastSnap := "" + if !containerOnly { + for i, snap := range s.rbdSnapshotNames { + prev := "" + if i > 0 { + prev = s.rbdSnapshotNames[i-1] + } + + lastSnap = snap + + sendSnapName := fmt.Sprintf( + "%s/container_%s@%s", + s.ceph.OSDPoolName, + project.Prefix(s.container.Project(), containerName), + snap) + + wrapper := StorageProgressReader(op, "fs_progress", snap) + + err := s.rbdSend( + conn, + sendSnapName, + prev, + wrapper) + if err != nil { + logger.Errorf(`Failed to send exported diff of RBD storage volume "%s" from snapshot "%s": %s`, sendSnapName, prev, err) + return err + } + logger.Debugf(`Sent exported diff of RBD storage volume "%s" from snapshot "%s"`, sendSnapName, prev) + } + } + + s.runningSnapName = fmt.Sprintf("migration-send-%s", uuid.NewRandom().String()) + err := cephRBDSnapshotCreate(s.ceph.ClusterName, s.ceph.OSDPoolName, + project.Prefix(s.container.Project(), containerName), storagePoolVolumeTypeNameContainer, + s.runningSnapName, s.ceph.UserName) + if err != nil { + logger.Errorf(`Failed to create snapshot "%s" for RBD storage volume for image "%s" on storage pool "%s": %s`, s.runningSnapName, containerName, s.ceph.pool.Name, err) + return err + } + + cur := fmt.Sprintf("%s/container_%s@%s", s.ceph.OSDPoolName, + project.Prefix(s.container.Project(), containerName), s.runningSnapName) + wrapper := StorageProgressReader(op, "fs_progress", containerName) + err = s.rbdSend(conn, cur, lastSnap, wrapper) + if err != nil { + logger.Errorf(`Failed to send exported diff of RBD storage volume "%s" from snapshot "%s": %s`, s.runningSnapName, lastSnap, err) + return err + } + logger.Debugf(`Sent exported diff of RBD storage volume "%s" from snapshot "%s"`, s.runningSnapName, lastSnap) + + return nil +} + +func (s *rbdMigrationSourceDriver) SendStorageVolume(conn *websocket.Conn, op *operation, bwlimit string, storage storage, volumeOnly bool) error { + msg := fmt.Sprintf("Function not implemented") + logger.Errorf(msg) + return fmt.Errorf(msg) +} + +// Let's say we want to send the a container "a" including snapshots "snap0" and +// "snap1" on storage pool "pool1" from LXD "l1" to LXD "l2" on storage pool +// "pool2": +// +// The pool layout on "l1" would be: +// pool1/container_a +// pool1/container_a@snapshot_snap0 +// pool1/container_a@snapshot_snap1 +// +// Then we need to send: +// rbd export-diff pool1/container_a@snapshot_snap0 - | rbd import-diff - pool2/container_a +// (Note that pool2/container_a must have been created by the receiving LXD +// instance before.) +// rbd export-diff pool1/container_a@snapshot_snap1 --from-snap snapshot_snap0 - | rbd import-diff - pool2/container_a +// rbd export-diff pool1/container_a --from-snap snapshot_snap1 - | rbd import-diff - pool2/container_a +func (s *rbdMigrationSourceDriver) rbdSend(conn *websocket.Conn, + volumeName string, + volumeParentName string, + readWrapper func(io.ReadCloser) io.ReadCloser) error { + args := []string{ + "export-diff", + "--cluster", s.ceph.ClusterName, + volumeName, + } + + if volumeParentName != "" { + args = append(args, "--from-snap", volumeParentName) + } + + // redirect output to stdout + args = append(args, "-") + + cmd := exec.Command("rbd", args...) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + + readPipe := io.ReadCloser(stdout) + if readWrapper != nil { + readPipe = readWrapper(stdout) + } + + stderr, err := cmd.StderrPipe() + if err != nil { + return err + } + + err = cmd.Start() + if err != nil { + return err + } + + <-shared.WebsocketSendStream(conn, readPipe, 4*1024*1024) + + output, err := ioutil.ReadAll(stderr) + if err != nil { + logger.Debugf(`Failed to read stderr output from "rbd export-diff": %s`, err) + } + + err = cmd.Wait() + if err != nil { + logger.Errorf(`Failed to perform "rbd export-diff": %s`, string(output)) + } + + return err +}
_______________________________________________ lxc-devel mailing list lxc-devel@lists.linuxcontainers.org http://lists.linuxcontainers.org/listinfo/lxc-devel