The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/6079
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) === Closes #5515
From b101362e97c2e41b94a5517cea5d283dc7288516 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Mon, 12 Aug 2019 00:14:38 -0400 Subject: [PATCH 1/5] api: Add daemon_storage extension MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- doc/api-extensions.md | 5 +++++ shared/version/api.go | 1 + 2 files changed, 6 insertions(+) diff --git a/doc/api-extensions.md b/doc/api-extensions.md index 06249be844..dc4ef31970 100644 --- a/doc/api-extensions.md +++ b/doc/api-extensions.md @@ -818,3 +818,8 @@ This makes use of shiftfs as an overlay filesystem. ## resources\_infiniband Export infiniband character device information (issm, umad, uverb) as part of the resources API. + +## daemon\_storage +This introduces two new configuration keys `storage.images\_volume` and +`storage.backups\_volume` to allow for a storage volume on an existing +pool be used for storing the daemon-wide images and backups artifacts. diff --git a/shared/version/api.go b/shared/version/api.go index 0e3f9c892e..201e834828 100644 --- a/shared/version/api.go +++ b/shared/version/api.go @@ -163,6 +163,7 @@ var APIExtensions = []string{ "container_disk_shift", "storage_shifted", "resources_infiniband", + "daemon_storage", } // APIExtensionsCount returns the number of available API extensions. From 4d37105c1a49315c82e01a351ffc9a2a56c19ff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Mon, 12 Aug 2019 00:14:52 -0400 Subject: [PATCH 2/5] doc: Add daemon storage keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- doc/server.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/server.md b/doc/server.md index 18628dd7c8..68d238c05c 100644 --- a/doc/server.md +++ b/doc/server.md @@ -46,6 +46,8 @@ rbac.agent.private\_key | string | global | - | rbac rbac.api.expiry | integer | global | - | rbac | RBAC macaroon expiry in seconds rbac.api.key | string | global | - | rbac | Public key of the RBAC server (required for HTTP-only servers) rbac.api.url | string | global | - | rbac | URL of the external RBAC server +storage.backups\_volume | string | local | - | daemon\_storage | Volume to use to store the backup tarballs (syntax is POOL/VOLUME) +storage.images\_volume | string | local | - | daemon\_storage | Volume to use to store the image tarballs (syntax is POOL/VOLUME) Those keys can be set using the lxc tool with: From 2366f593189df4802ac6bf30f5cba8167d195f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Mon, 12 Aug 2019 00:15:32 -0400 Subject: [PATCH 3/5] bash: Add daemon storage keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- scripts/bash/lxd-client | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/bash/lxd-client b/scripts/bash/lxd-client index b8613c4e0c..6a78584688 100644 --- a/scripts/bash/lxd-client +++ b/scripts/bash/lxd-client @@ -75,7 +75,8 @@ _have lxc && { images.compression_algorithm images.remote_cache_expiry \ maas.api.url maas.api.key maas.machine cluster.images_minimal_replica \ rbac.agent.url rbac.agent.username rbac.agent.public_key \ - rbac.agent.private_key rbac.api.expiry rbac.api.key rbac.api.url" + rbac.agent.private_key rbac.api.expiry rbac.api.key rbac.api.url \ + storage.backups_volume storage.images_volume" container_keys="boot.autostart boot.autostart.delay \ boot.autostart.priority boot.stop.priority \ From a50b8608f11bac5f6d72d17561c29eef78644ae6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Mon, 12 Aug 2019 00:27:28 -0400 Subject: [PATCH 4/5] lxd: Add daemon storage keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- lxd/node/config.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lxd/node/config.go b/lxd/node/config.go index c67814c030..c478791faa 100644 --- a/lxd/node/config.go +++ b/lxd/node/config.go @@ -54,6 +54,16 @@ func (c *Config) MAASMachine() string { return c.m.GetString("maas.machine") } +// StorageBackupsVolume returns the name of the pool/volume to use for storing backup tarballs +func (c *Config) StorageBackupsVolume() string { + return c.m.GetString("storage.backups_volume") +} + +// StorageImagesVolume returns the name of the pool/volume to use for storing image tarballs +func (c *Config) StorageImagesVolume() string { + return c.m.GetString("storage.images_volume") +} + // Dump current configuration keys and their values. Keys with values matching // their defaults are omitted. func (c *Config) Dump() map[string]interface{} { @@ -150,4 +160,8 @@ var ConfigSchema = config.Schema{ // MAAS machine this LXD instance is associated with "maas.machine": {}, + + // Storage volumes to store backups/images on + "storage.backups_volume": {}, + "storage.images_volume": {}, } From 29a065245b7529ba20f941d512c301101f879d13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Tue, 13 Aug 2019 22:45:57 -0400 Subject: [PATCH 5/5] lxd/daemon: Support storing images/backups on pool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- lxd/api_1.0.go | 37 +++- lxd/daemon.go | 6 + lxd/daemon_storage.go | 348 ++++++++++++++++++++++++++++++++ lxd/storage_volumes.go | 19 +- lxd/storage_volumes_snapshot.go | 14 +- lxd/storage_volumes_utils.go | 10 + 6 files changed, 426 insertions(+), 8 deletions(-) create mode 100644 lxd/daemon_storage.go diff --git a/lxd/api_1.0.go b/lxd/api_1.0.go index c794c2ff6f..0b6e1a9db4 100644 --- a/lxd/api_1.0.go +++ b/lxd/api_1.0.go @@ -321,6 +321,8 @@ func api10Patch(d *Daemon, r *http.Request) Response { } func doApi10Update(d *Daemon, req api.ServerPut, patch bool) Response { + s := d.State() + // First deal with config specific to the local daemon nodeValues := map[string]interface{}{} @@ -350,6 +352,21 @@ func doApi10Update(d *Daemon, req api.ServerPut, patch bool) Response { return fmt.Errorf("Changing cluster.https_address is currently not supported") } + // Validate the storage volumes + if nodeValues["storage.backups_volume"] != nil && nodeValues["storage.backups_volume"] != newNodeConfig.StorageBackupsVolume() { + err := daemonStorageValidate(s, nodeValues["storage.backups_volume"].(string)) + if err != nil { + return err + } + } + + if nodeValues["storage.images_volume"] != nil && nodeValues["storage.images_volume"] != newNodeConfig.StorageImagesVolume() { + err := daemonStorageValidate(s, nodeValues["storage.images_volume"].(string)) + if err != nil { + return err + } + } + if patch { nodeChanged, err = newNodeConfig.Patch(nodeValues) } else { @@ -411,7 +428,7 @@ func doApi10Update(d *Daemon, req api.ServerPut, patch bool) Response { } // Notify the other nodes about changes - notifier, err := cluster.NewNotifier(d.State(), d.endpoints.NetworkCert(), cluster.NotifyAlive) + notifier, err := cluster.NewNotifier(s, d.endpoints.NetworkCert(), cluster.NotifyAlive) if err != nil { return SmartError(err) } @@ -442,6 +459,8 @@ func doApi10Update(d *Daemon, req api.ServerPut, patch bool) Response { } func doApi10UpdateTriggers(d *Daemon, nodeChanged, clusterChanged map[string]string, nodeConfig *node.Config, clusterConfig *cluster.Config) error { + s := d.State() + maasChanged := false candidChanged := false rbacChanged := false @@ -525,6 +544,22 @@ func doApi10UpdateTriggers(d *Daemon, nodeChanged, clusterChanged map[string]str } } + value, ok = nodeChanged["storage.backups_volume"] + if ok { + err := daemonStorageMove(s, "backups", value) + if err != nil { + return err + } + } + + value, ok = nodeChanged["storage.images_volume"] + if ok { + err := daemonStorageMove(s, "images", value) + if err != nil { + return err + } + } + if maasChanged { url, key := clusterConfig.MAASController() machine := nodeConfig.MAASMachine() diff --git a/lxd/daemon.go b/lxd/daemon.go index 64b2abb3de..0aad1da9f9 100644 --- a/lxd/daemon.go +++ b/lxd/daemon.go @@ -777,6 +777,12 @@ func (d *Daemon) init() error { return err } + // Mount any daemon storage + err = daemonStorageMount(d.State()) + if err != nil { + return err + } + /* Apply all patches */ err = patchesApplyAll(d) if err != nil { diff --git a/lxd/daemon_storage.go b/lxd/daemon_storage.go new file mode 100644 index 0000000000..ef55306190 --- /dev/null +++ b/lxd/daemon_storage.go @@ -0,0 +1,348 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" + + "github.com/lxc/lxd/lxd/db" + "github.com/lxc/lxd/lxd/node" + "github.com/lxc/lxd/lxd/state" + "github.com/lxc/lxd/shared" + "github.com/lxc/lxd/shared/logger" +) + +func daemonStorageMount(s *state.State) error { + var storageBackups string + var storageImages string + err := s.Node.Transaction(func(tx *db.NodeTx) error { + nodeConfig, err := node.ConfigLoad(tx) + if err != nil { + return err + } + + storageBackups = nodeConfig.StorageBackupsVolume() + storageImages = nodeConfig.StorageImagesVolume() + + return nil + }) + if err != nil { + return err + } + + mount := func(storageType string, source string) error { + // Parse the source + fields := strings.Split(source, "/") + if len(fields) != 2 { + return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>") + } + + poolName := fields[0] + volumeName := fields[1] + + // Mount volume + volume, err := storageInit(s, "default", poolName, volumeName, storagePoolVolumeTypeCustom) + if err != nil { + return errors.Wrapf(err, "Unable to load storage volume \"%s\"", source) + } + + _, err = volume.StoragePoolVolumeMount() + if err != nil { + return errors.Wrapf(err, "Failed to mount storage volume \"%s\"", source) + } + + return nil + } + + if storageBackups != "" { + err := mount("backups", storageBackups) + if err != nil { + return errors.Wrap(err, "Failed to mount backups storage") + } + } + + if storageImages != "" { + err := mount("images", storageImages) + if err != nil { + return errors.Wrap(err, "Failed to mount images storage") + } + } + + return nil +} + +func daemonStorageUsed(s *state.State, poolName string, volumeName string) (bool, error) { + var storageBackups string + var storageImages string + err := s.Node.Transaction(func(tx *db.NodeTx) error { + nodeConfig, err := node.ConfigLoad(tx) + if err != nil { + return err + } + + storageBackups = nodeConfig.StorageBackupsVolume() + storageImages = nodeConfig.StorageImagesVolume() + + return nil + }) + if err != nil { + return false, err + } + + fullName := fmt.Sprintf("%s/%s", poolName, volumeName) + if storageBackups == fullName || storageImages == fullName { + return true, nil + } + + return false, nil +} + +func daemonStorageValidate(s *state.State, target string) error { + // Check syntax + if target == "" { + return nil + } + + fields := strings.Split(target, "/") + if len(fields) != 2 { + return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>") + } + + poolName := fields[0] + volumeName := fields[1] + + // Validate pool exists + poolID, pool, err := s.Cluster.StoragePoolGet(poolName) + if err != nil { + return errors.Wrapf(err, "Unable to load storage pool \"%s\"", poolName) + } + + // Validate pool driver (can't be CEPH or CEPHFS) + if pool.Driver == "ceph" || pool.Driver == "cephfs" { + return fmt.Errorf("Server storage volumes cannot be stored on Ceph") + } + + // Confirm volume exists + volume, err := storageInit(s, "default", poolName, volumeName, storagePoolVolumeTypeCustom) + if err != nil { + return errors.Wrapf(err, "Unable to load storage volume \"%s\"", target) + } + + snapshots, err := s.Cluster.StoragePoolVolumeSnapshotsGetType(volumeName, storagePoolVolumeTypeCustom, poolID) + if err != nil { + return errors.Wrapf(err, "Unable to load storage volume snapshots \"%s\"", target) + } + + if len(snapshots) != 0 { + return fmt.Errorf("Storage volumes for use by LXD itself cannot have snapshots") + } + + // Mount volume + ourMount, err := volume.StoragePoolVolumeMount() + if err != nil { + return errors.Wrapf(err, "Failed to mount storage volume \"%s\"", target) + } + if ourMount { + defer volume.StoragePoolUmount() + } + + // Validate volume is empty (ignore lost+found) + mountpoint := shared.VarPath("storage-pools", poolName, "custom", volumeName) + + entries, err := ioutil.ReadDir(mountpoint) + if err != nil { + return errors.Wrapf(err, "Failed to list \"%s\"", mountpoint) + } + + for _, entry := range entries { + entryName := entry.Name() + + if entryName == "lost+found" { + continue + } + + return fmt.Errorf("Storage volume \"%s\" isn't empty", target) + } + + return nil +} + +func daemonStorageMove(s *state.State, storageType string, target string) error { + logger.Errorf("stgraber: daemonStorageMove for %s at %s", storageType, target) + destPath := shared.VarPath(storageType) + + // Track down the current storage + var sourcePool string + var sourceVolume string + + sourcePath, err := os.Readlink(destPath) + if err != nil { + sourcePath = destPath + } else { + fields := strings.Split(sourcePath, "/") + sourcePool = fields[len(fields)-3] + sourceVolume = fields[len(fields)-1] + } + + moveContent := func(source string, target string) error { + // Copy the content + _, err := rsyncLocalCopy(source, target, "", false) + if err != nil { + return err + } + + // Remove the source content + entries, err := ioutil.ReadDir(source) + if err != nil { + return err + } + + for _, entry := range entries { + err := os.RemoveAll(filepath.Join(source, entry.Name())) + if err != nil { + return err + } + } + + return nil + } + + // Deal with unsetting + if target == "" { + // Things already look correct + if sourcePath == destPath { + return nil + } + + // Remove the symlink + err = os.Remove(destPath) + if err != nil { + return errors.Wrapf(err, "Failed to delete storage symlink at \"%s\"", destPath) + } + + // Re-create as a directory + err = os.MkdirAll(destPath, 0700) + if err != nil { + return errors.Wrapf(err, "Failed to create directory \"%s\"", destPath) + } + + // Move the data across + err = moveContent(sourcePath, destPath) + if err != nil { + return errors.Wrapf(err, "Failed to move data over to directory \"%s\"", destPath) + } + + // Unmount old volume + volume, err := storageInit(s, "default", sourcePool, sourceVolume, storagePoolVolumeTypeCustom) + if err != nil { + return errors.Wrapf(err, "Unable to load storage volume \"%s/%s\"", sourcePool, sourceVolume) + } + + _, err = volume.StoragePoolVolumeUmount() + if err != nil { + return errors.Wrapf(err, "Failed to umount storage volume \"%s/%s\"", sourcePool, sourceVolume) + } + + return nil + } + + // Parse the target + fields := strings.Split(target, "/") + if len(fields) != 2 { + return fmt.Errorf("Invalid syntax for volume, must be <pool>/<volume>") + } + + poolName := fields[0] + volumeName := fields[1] + + // Mount volume + volume, err := storageInit(s, "default", poolName, volumeName, storagePoolVolumeTypeCustom) + if err != nil { + return errors.Wrapf(err, "Unable to load storage volume \"%s\"", target) + } + + _, err = volume.StoragePoolVolumeMount() + if err != nil { + return errors.Wrapf(err, "Failed to mount storage volume \"%s\"", target) + } + + // Set ownership & mode + mountpoint := shared.VarPath("storage-pools", poolName, "custom", volumeName) + destPath = mountpoint + + err = os.Chmod(mountpoint, 0700) + if err != nil { + return errors.Wrapf(err, "Failed to set permissions on \"%s\"", mountpoint) + } + + err = os.Chown(mountpoint, 0, 0) + if err != nil { + return errors.Wrapf(err, "Failed to set ownership on \"%s\"", mountpoint) + } + + // Handle changes + if sourcePath != shared.VarPath(storageType) { + // Remove the symlink + err := os.Remove(shared.VarPath(storageType)) + if err != nil { + return errors.Wrapf(err, "Failed to remove the new symlink at \"%s\"", shared.VarPath(storageType)) + } + + // Create the new symlink + err = os.Symlink(destPath, shared.VarPath(storageType)) + if err != nil { + return errors.Wrapf(err, "Failed to create the new symlink at \"%s\"", shared.VarPath(storageType)) + } + + // Move the data across + err = moveContent(sourcePath, destPath) + if err != nil { + return errors.Wrapf(err, "Failed to move data over to directory \"%s\"", destPath) + } + + // Unmount old volume + volume, err := storageInit(s, "default", sourcePool, sourceVolume, storagePoolVolumeTypeCustom) + if err != nil { + return errors.Wrapf(err, "Unable to load storage volume \"%s/%s\"", sourcePool, sourceVolume) + } + + _, err = volume.StoragePoolVolumeUmount() + if err != nil { + return errors.Wrapf(err, "Failed to umount storage volume \"%s/%s\"", sourcePool, sourceVolume) + } + + return nil + } + + sourcePath = shared.VarPath(storageType) + ".temp" + + // Rename the existing storage + err = os.Rename(shared.VarPath(storageType), sourcePath) + if err != nil { + return errors.Wrapf(err, "Failed to rename existing storage \"%s\"", shared.VarPath(storageType)) + } + + // Create the new symlink + err = os.Symlink(destPath, shared.VarPath(storageType)) + if err != nil { + return errors.Wrapf(err, "Failed to create the new symlink at \"%s\"", shared.VarPath(storageType)) + } + + // Move the data across + err = moveContent(sourcePath, destPath) + if err != nil { + return errors.Wrapf(err, "Failed to move data over to directory \"%s\"", destPath) + } + + // Remove the old data + err = os.RemoveAll(sourcePath) + if err != nil { + return errors.Wrapf(err, "Failed to cleanup old directory \"%s\"", sourcePath) + } + + return nil +} diff --git a/lxd/storage_volumes.go b/lxd/storage_volumes.go index 38250235d0..ce02541471 100644 --- a/lxd/storage_volumes.go +++ b/lxd/storage_volumes.go @@ -452,7 +452,7 @@ func storagePoolVolumeTypePost(d *Daemon, r *http.Request, volumeTypeName string // Handle volume volumeName = fields[0] } else { - return BadRequest(fmt.Errorf("invalid storage volume %s", mux.Vars(r)["name"])) + return BadRequest(fmt.Errorf("Invalid storage volume %s", mux.Vars(r)["name"])) } // Get the name of the storage pool the volume is supposed to be @@ -560,8 +560,7 @@ func storagePoolVolumeTypePost(d *Daemon, r *http.Request, volumeTypeName string } // Check that the name isn't already in use. - _, err = d.cluster.StoragePoolNodeVolumeGetTypeID(req.Name, - storagePoolVolumeTypeCustom, poolID) + _, err = d.cluster.StoragePoolNodeVolumeGetTypeID(req.Name, storagePoolVolumeTypeCustom, poolID) if err != db.ErrNoSuchObject { if err != nil { return InternalError(err) @@ -571,6 +570,17 @@ func storagePoolVolumeTypePost(d *Daemon, r *http.Request, volumeTypeName string } doWork := func() error { + // Check if the daemon itself is using it + used, err := daemonStorageUsed(d.State(), poolName, volumeName) + if err != nil { + return err + } + + if used { + return fmt.Errorf("Volume is used by LXD itself and cannot be renamed") + } + + // Check if a running container is using it ctsUsingVolume, err := storagePoolVolumeUsedByRunningContainersWithProfilesGet(d.State(), poolName, volumeName, storagePoolVolumeTypeNameCustom, true) if err != nil { return err @@ -1007,8 +1017,7 @@ func storagePoolVolumeTypeDelete(d *Daemon, r *http.Request, volumeTypeName stri "/%s/images/%s", version.APIVersion, volumeName) { - return BadRequest(fmt.Errorf(`The storage volume is ` + - `still in use by containers or profiles`)) + return BadRequest(fmt.Errorf("The storage volume is still in use")) } } diff --git a/lxd/storage_volumes_snapshot.go b/lxd/storage_volumes_snapshot.go index 3b2b2cfa10..8e87a6e925 100644 --- a/lxd/storage_volumes_snapshot.go +++ b/lxd/storage_volumes_snapshot.go @@ -56,7 +56,7 @@ func storagePoolVolumeSnapshotsTypePost(d *Daemon, r *http.Request) Response { // Check that the storage volume type is valid. if !shared.IntInSlice(volumeType, supportedVolumeTypes) { - return BadRequest(fmt.Errorf("invalid storage volume type \"%d\"", volumeType)) + return BadRequest(fmt.Errorf("Invalid storage volume type \"%d\"", volumeType)) } // Get a snapshot name. @@ -71,6 +71,16 @@ func storagePoolVolumeSnapshotsTypePost(d *Daemon, r *http.Request) Response { return BadRequest(err) } + // Check that this isn't a restricted volume + used, err := daemonStorageUsed(d.State(), poolName, volumeName) + if err != nil { + return InternalError(err) + } + + if used { + return BadRequest(fmt.Errorf("Volumes used by LXD itself cannot have snapshots")) + } + // Retrieve ID of the storage pool (and check if the storage pool // exists). poolID, err := d.cluster.StoragePoolGetID(poolName) @@ -94,7 +104,7 @@ func storagePoolVolumeSnapshotsTypePost(d *Daemon, r *http.Request) Response { return SmartError(err) } - // Ensure that it doens't already fucking exist + // Ensure that the snapshot doens't already exist _, _, err = d.cluster.StoragePoolNodeVolumeGetType(fmt.Sprintf("%s/%s", volumeName, req.Name), volumeType, poolID) if err != db.ErrNoSuchObject { if err != nil { diff --git a/lxd/storage_volumes_utils.go b/lxd/storage_volumes_utils.go index 538b4704fe..f7d32012b0 100644 --- a/lxd/storage_volumes_utils.go +++ b/lxd/storage_volumes_utils.go @@ -447,6 +447,16 @@ func storagePoolVolumeUsedByGet(s *state.State, project, poolName string, volume return []string{fmt.Sprintf("/%s/images/%s", version.APIVersion, volumeName)}, nil } + // Check if the daemon itself is using it + used, err := daemonStorageUsed(s, poolName, volumeName) + if err != nil { + return []string{}, err + } + + if used { + return []string{fmt.Sprintf("/%s", version.APIVersion)}, nil + } + // Look for containers using this volume ctsUsingVolume, err := storagePoolVolumeUsedByContainersGet(s, project, poolName, volumeName) if err != nil {
_______________________________________________ lxc-devel mailing list lxc-devel@lists.linuxcontainers.org http://lists.linuxcontainers.org/listinfo/lxc-devel