This is an automated email from the ASF dual-hosted git repository. jpeach pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/mesos.git
commit 8a47fdd86c1ca9536eeca4f23b6cac5bd8ea4f2d Author: James Peach <[email protected]> AuthorDate: Wed Aug 7 20:21:40 2019 -0700 Propagate ephemeral volume information from rootfs. Propagate the overlayfs upperdir path to the containerization layer in a general form as a set of ephemeral volume paths. Review: https://reviews.apache.org/r/71192/ --- include/mesos/slave/containerizer.proto | 6 ++++++ src/slave/containerizer/mesos/containerizer.cpp | 18 +++++++++++++++++- src/slave/containerizer/mesos/provisioner/backend.hpp | 7 ++++++- .../containerizer/mesos/provisioner/backends/aufs.cpp | 8 ++++---- .../containerizer/mesos/provisioner/backends/aufs.hpp | 2 +- .../containerizer/mesos/provisioner/backends/bind.cpp | 9 +++++---- .../containerizer/mesos/provisioner/backends/bind.hpp | 2 +- .../containerizer/mesos/provisioner/backends/copy.cpp | 9 +++++---- .../containerizer/mesos/provisioner/backends/copy.hpp | 2 +- .../mesos/provisioner/backends/overlay.cpp | 11 +++++++---- .../mesos/provisioner/backends/overlay.hpp | 2 +- .../containerizer/mesos/provisioner/provisioner.cpp | 5 +++-- .../containerizer/mesos/provisioner/provisioner.hpp | 5 +++++ 13 files changed, 62 insertions(+), 24 deletions(-) diff --git a/include/mesos/slave/containerizer.proto b/include/mesos/slave/containerizer.proto index a60c963..3e5b667 100644 --- a/include/mesos/slave/containerizer.proto +++ b/include/mesos/slave/containerizer.proto @@ -79,6 +79,9 @@ message ContainerState { // The sandbox directory. required string directory = 4; + + // Ephemeral path volumes subject to container disk quota. + repeated string ephemeral_volumes = 5; } @@ -142,6 +145,9 @@ message ContainerConfig { // The work directory for the container in the host filesystem. required string directory = 3; + // Ephemeral path volumes subject to container disk quota. + repeated string ephemeral_volumes = 15; + // The user that should be used to run the `command_info`. // The sandbox directory and any artifacts from the Mesos fetcher will // be made accessible to this user. diff --git a/src/slave/containerizer/mesos/containerizer.cpp b/src/slave/containerizer/mesos/containerizer.cpp index a01edc8..75e7cda 100644 --- a/src/slave/containerizer/mesos/containerizer.cpp +++ b/src/slave/containerizer/mesos/containerizer.cpp @@ -904,7 +904,7 @@ Future<Nothing> MesosContainerizerProcess::recover( } // Recover the containers from 'SlaveState'. - foreach (const ContainerState& state, recoverable) { + foreach (ContainerState& state, recoverable) { const ContainerID& containerId = state.container_id(); // Contruct the structure for containers from the 'SlaveState' @@ -931,6 +931,10 @@ Future<Nothing> MesosContainerizerProcess::recover( if (config.isSome()) { container->config = config.get(); + + // Copy the ephemeral volume paths to the ContainerState, since this + // information is otherwise only available to the prepare() callback. + *state.mutable_ephemeral_volumes() = config->ephemeral_volumes(); } else { VLOG(1) << "No config is recovered for container " << containerId << ", this means image pruning will be disabled."; @@ -1094,6 +1098,12 @@ Future<Nothing> MesosContainerizerProcess::recover( container->pid.get(), container->directory.get()); + if (config.isSome()) { + // Copy the ephemeral volume paths to the ContainerState, since this + // information is otherwise only available to the prepare() callback. + *state.mutable_ephemeral_volumes() = config->ephemeral_volumes(); + } + recoverable.push_back(state); continue; } @@ -1517,6 +1527,12 @@ Future<Nothing> MesosContainerizerProcess::prepare( if (provisionInfo.isSome()) { container->config->set_rootfs(provisionInfo->rootfs); + if (provisionInfo->ephemeralVolumes.isSome()) { + foreach (const Path& path, provisionInfo->ephemeralVolumes.get()) { + container->config->add_ephemeral_volumes(path); + } + } + if (provisionInfo->dockerManifest.isSome() && provisionInfo->appcManifest.isSome()) { return Failure("Container cannot have both Docker and Appc manifests"); diff --git a/src/slave/containerizer/mesos/provisioner/backend.hpp b/src/slave/containerizer/mesos/provisioner/backend.hpp index 7257d3a..55f007f 100644 --- a/src/slave/containerizer/mesos/provisioner/backend.hpp +++ b/src/slave/containerizer/mesos/provisioner/backend.hpp @@ -24,6 +24,8 @@ #include <process/owned.hpp> #include <stout/hashmap.hpp> +#include <stout/option.hpp> +#include <stout/path.hpp> #include <stout/try.hpp> #include "slave/flags.hpp" @@ -48,7 +50,10 @@ public: // directory by applying the specified list of root filesystem layers in // the list order, i.e., files in a layer can overwrite/shadow those from // another layer earlier in the list. - virtual process::Future<Nothing> provision( + // + // Optionally returns a set of paths whose contents should be included + // in the ephemeral sandbox disk quota. + virtual process::Future<Option<std::vector<Path>>> provision( const std::vector<std::string>& layers, const std::string& rootfs, const std::string& backendDir) = 0; diff --git a/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp b/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp index 2eba552..e401e5b 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp +++ b/src/slave/containerizer/mesos/provisioner/backends/aufs.cpp @@ -51,7 +51,7 @@ public: AufsBackendProcess() : ProcessBase(process::ID::generate("aufs-provisioner-backend")) {} - Future<Nothing> provision( + Future<Option<vector<Path>>> provision( const vector<string>& layers, const string& rootfs, const string& backendDir); @@ -87,7 +87,7 @@ AufsBackend::AufsBackend(Owned<AufsBackendProcess> _process) } -Future<Nothing> AufsBackend::provision( +Future<Option<vector<Path>>> AufsBackend::provision( const vector<string>& layers, const string& rootfs, const string& backendDir) @@ -113,7 +113,7 @@ Future<bool> AufsBackend::destroy( } -Future<Nothing> AufsBackendProcess::provision( +Future<Option<vector<Path>>> AufsBackendProcess::provision( const vector<string>& layers, const string& rootfs, const string& backendDir) @@ -236,7 +236,7 @@ Future<Nothing> AufsBackendProcess::provision( "' as a shared mount: " + mount.error()); } - return Nothing(); + return None(); } diff --git a/src/slave/containerizer/mesos/provisioner/backends/aufs.hpp b/src/slave/containerizer/mesos/provisioner/backends/aufs.hpp index 2c25187..8a08339 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/aufs.hpp +++ b/src/slave/containerizer/mesos/provisioner/backends/aufs.hpp @@ -48,7 +48,7 @@ public: static Try<process::Owned<Backend>> create(const Flags&); - process::Future<Nothing> provision( + process::Future<Option<std::vector<Path>>> provision( const std::vector<std::string>& layers, const std::string& rootfs, const std::string& backendDir) override; diff --git a/src/slave/containerizer/mesos/provisioner/backends/bind.cpp b/src/slave/containerizer/mesos/provisioner/backends/bind.cpp index 4cc4c52..e906a88 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/bind.cpp +++ b/src/slave/containerizer/mesos/provisioner/backends/bind.cpp @@ -47,7 +47,8 @@ public: BindBackendProcess() : ProcessBase(process::ID::generate("bind-provisioner-backend")) {} - Future<Nothing> provision(const vector<string>& layers, const string& rootfs); + Future<Option<vector<Path>>> provision( + const vector<string>& layers, const string& rootfs); Future<bool> destroy(const string& rootfs); @@ -86,7 +87,7 @@ BindBackend::BindBackend(Owned<BindBackendProcess> _process) } -Future<Nothing> BindBackend::provision( +Future<Option<std::vector<Path>>> BindBackend::provision( const vector<string>& layers, const string& rootfs, const string& backendDir) @@ -104,7 +105,7 @@ Future<bool> BindBackend::destroy( } -Future<Nothing> BindBackendProcess::provision( +Future<Option<std::vector<Path>>> BindBackendProcess::provision( const vector<string>& layers, const string& rootfs) { @@ -164,7 +165,7 @@ Future<Nothing> BindBackendProcess::provision( "' as a shared mount: " + mount.error()); } - return Nothing(); + return None(); } diff --git a/src/slave/containerizer/mesos/provisioner/backends/bind.hpp b/src/slave/containerizer/mesos/provisioner/backends/bind.hpp index c4a1d5f..9c908fd 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/bind.hpp +++ b/src/slave/containerizer/mesos/provisioner/backends/bind.hpp @@ -55,7 +55,7 @@ public: // BindBackend doesn't use any flag. static Try<process::Owned<Backend>> create(const Flags&); - process::Future<Nothing> provision( + process::Future<Option<std::vector<Path>>> provision( const std::vector<std::string>& layers, const std::string& rootfs, const std::string& backendDir) override; diff --git a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp b/src/slave/containerizer/mesos/provisioner/backends/copy.cpp index 10516ca..4afef7f 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp +++ b/src/slave/containerizer/mesos/provisioner/backends/copy.cpp @@ -52,7 +52,8 @@ public: CopyBackendProcess() : ProcessBase(process::ID::generate("copy-provisioner-backend")) {} - Future<Nothing> provision(const vector<string>& layers, const string& rootfs); + Future<Option<vector<Path>>> provision( + const vector<string>& layers, const string& rootfs); Future<bool> destroy(const string& rootfs); @@ -82,7 +83,7 @@ CopyBackend::CopyBackend(Owned<CopyBackendProcess> _process) } -Future<Nothing> CopyBackend::provision( +Future<Option<vector<Path>>> CopyBackend::provision( const vector<string>& layers, const string& rootfs, const string& backendDir) @@ -100,7 +101,7 @@ Future<bool> CopyBackend::destroy( } -Future<Nothing> CopyBackendProcess::provision( +Future<Option<vector<Path>>> CopyBackendProcess::provision( const vector<string>& layers, const string& rootfs) { @@ -126,7 +127,7 @@ Future<Nothing> CopyBackendProcess::provision( } return collect(futures) - .then([]() -> Future<Nothing> { return Nothing(); }); + .then([]() -> Future<Option<vector<Path>>> { return None(); }); } diff --git a/src/slave/containerizer/mesos/provisioner/backends/copy.hpp b/src/slave/containerizer/mesos/provisioner/backends/copy.hpp index 5dc9a2f..9e8bd97 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/copy.hpp +++ b/src/slave/containerizer/mesos/provisioner/backends/copy.hpp @@ -45,7 +45,7 @@ public: // Provisions a rootfs given the layers' paths and target rootfs // path. - process::Future<Nothing> provision( + process::Future<Option<std::vector<Path>>> provision( const std::vector<std::string>& layers, const std::string& rootfs, const std::string& backendDir) override; diff --git a/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp b/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp index f2040cf..77d6711 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp +++ b/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp @@ -52,7 +52,7 @@ public: OverlayBackendProcess() : ProcessBase(process::ID::generate("overlay-provisioner-backend")) {} - Future<Nothing> provision( + Future<Option<vector<Path>>> provision( const vector<string>& layers, const string& rootfs, const string& backendDir); @@ -88,7 +88,7 @@ OverlayBackend::OverlayBackend(Owned<OverlayBackendProcess> _process) } -Future<Nothing> OverlayBackend::provision( +Future<Option<vector<Path>>> OverlayBackend::provision( const vector<string>& layers, const string& rootfs, const string& backendDir) @@ -114,7 +114,7 @@ Future<bool> OverlayBackend::destroy( } -Future<Nothing> OverlayBackendProcess::provision( +Future<Option<vector<Path>>> OverlayBackendProcess::provision( const vector<string>& layers, const string& rootfs, const string& backendDir) @@ -237,7 +237,10 @@ Future<Nothing> OverlayBackendProcess::provision( "' as a shared mount: " + mount.error()); } - return Nothing(); + // Note that both upperdir and workdir are ephemeral. The `disk/xfs` + // isolator needs this because XFS will error with EXDEV when renaming + // a file into a tree with a different project ID (see xfs_rename). + return vector<Path>{Path(upperdir), Path(workdir)}; } diff --git a/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp b/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp index 362e021..78896b6 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp +++ b/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp @@ -49,7 +49,7 @@ public: static Try<process::Owned<Backend>> create(const Flags&); - process::Future<Nothing> provision( + process::Future<Option<std::vector<Path>>> provision( const std::vector<std::string>& layers, const std::string& rootfs, const std::string& backendDir) override; diff --git a/src/slave/containerizer/mesos/provisioner/provisioner.cpp b/src/slave/containerizer/mesos/provisioner/provisioner.cpp index a081fb0..bf3908d 100644 --- a/src/slave/containerizer/mesos/provisioner/provisioner.cpp +++ b/src/slave/containerizer/mesos/provisioner/provisioner.cpp @@ -574,7 +574,8 @@ Future<ProvisionInfo> ProvisionerProcess::_provision( imageInfo.layers, rootfs, backendDir) - .then(defer(self(), [=]() -> Future<ProvisionInfo> { + .then(defer(self(), [=](const Option<vector<Path>>& ephemeral) + -> Future<ProvisionInfo> { const string path = provisioner::paths::getLayersFilePath(rootDir, containerId); @@ -596,7 +597,7 @@ Future<ProvisionInfo> ProvisionerProcess::_provision( } return ProvisionInfo{ - rootfs, imageInfo.dockerManifest, imageInfo.appcManifest}; + rootfs, ephemeral, imageInfo.dockerManifest, imageInfo.appcManifest}; })); } diff --git a/src/slave/containerizer/mesos/provisioner/provisioner.hpp b/src/slave/containerizer/mesos/provisioner/provisioner.hpp index 7f84aa4..3866417 100644 --- a/src/slave/containerizer/mesos/provisioner/provisioner.hpp +++ b/src/slave/containerizer/mesos/provisioner/provisioner.hpp @@ -30,6 +30,7 @@ #include <mesos/slave/isolator.hpp> // For ContainerState. #include <stout/nothing.hpp> +#include <stout/path.hpp> #include <stout/try.hpp> #include <process/future.hpp> @@ -61,6 +62,10 @@ struct ProvisionInfo { std::string rootfs; + // Ephemeral volumes are any additional paths the Provisioner backend + // may have created that should be counted towards the sandbox disk quota. + Option<std::vector<Path>> ephemeralVolumes; + // Docker v1 image manifest. Option<::docker::spec::v1::ImageManifest> dockerManifest;
