Repository: mesos Updated Branches: refs/heads/master d353bcb16 -> 4dfa91fc2
Improved overlay backend to make the rootfs writable. Review: https://reviews.apache.org/r/45358/ Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/4dfa91fc Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/4dfa91fc Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/4dfa91fc Branch: refs/heads/master Commit: 4dfa91fc21f80204f5125b2e2f35c489f8fb41d8 Parents: d353bcb Author: Shuai Lin <linshuai2...@gmail.com> Authored: Mon Apr 11 08:51:31 2016 -0700 Committer: Jie Yu <yujie....@gmail.com> Committed: Mon Apr 11 09:17:38 2016 -0700 ---------------------------------------------------------------------- docs/container-image.md | 10 --- .../containerizer/mesos/provisioner/backend.hpp | 3 +- .../mesos/provisioner/backends/bind.cpp | 3 +- .../mesos/provisioner/backends/bind.hpp | 3 +- .../mesos/provisioner/backends/copy.cpp | 3 +- .../mesos/provisioner/backends/copy.hpp | 3 +- .../mesos/provisioner/backends/overlay.cpp | 69 ++++++++++++++------ .../mesos/provisioner/backends/overlay.hpp | 21 ++++-- .../containerizer/mesos/provisioner/paths.cpp | 13 ++++ .../containerizer/mesos/provisioner/paths.hpp | 7 ++ .../mesos/provisioner/provisioner.cpp | 10 ++- .../containerizer/provisioner_backend_tests.cpp | 48 ++++++++------ 12 files changed, 130 insertions(+), 63 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/docs/container-image.md ---------------------------------------------------------------------- diff --git a/docs/container-image.md b/docs/container-image.md index c4abfa6..843881e 100644 --- a/docs/container-image.md +++ b/docs/container-image.md @@ -298,16 +298,6 @@ for more detail. For more information of overlayfs, please refer to [here](https://www.kernel.org/doc/Documentation/filesystems/overlayfs.txt). -The overlay backend currently has these two limitations: - -1. The overlay backend only supports more than one single layer. One -single layer images will fail to provision and the container will fail -to launch! This limitation will be solved soon. - -2. Similar to the bind backend, the filesystem will be read-only. -Please refer to the second limitation of the bind backend for more -details. We will resolve this limitation soon. - ## Executor Dependencies in a Container Image http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backend.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/backend.hpp b/src/slave/containerizer/mesos/provisioner/backend.hpp index c6cca81..93819c8 100644 --- a/src/slave/containerizer/mesos/provisioner/backend.hpp +++ b/src/slave/containerizer/mesos/provisioner/backend.hpp @@ -50,7 +50,8 @@ public: // another layer earlier in the list. virtual process::Future<Nothing> provision( const std::vector<std::string>& layers, - const std::string& rootfs) = 0; + const std::string& rootfs, + const std::string& backendDir) = 0; // Destroy the root filesystem provisioned at the specified 'rootfs' // directory. Return false if there is no provisioned root filesystem http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backends/bind.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/backends/bind.cpp b/src/slave/containerizer/mesos/provisioner/backends/bind.cpp index 9b9f0b9..ea7829b 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/bind.cpp +++ b/src/slave/containerizer/mesos/provisioner/backends/bind.cpp @@ -90,7 +90,8 @@ BindBackend::BindBackend(Owned<BindBackendProcess> _process) Future<Nothing> BindBackend::provision( const vector<string>& layers, - const string& rootfs) + const string& rootfs, + const string& backendDir) { return dispatch( process.get(), &BindBackendProcess::provision, layers, rootfs); http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backends/bind.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/backends/bind.hpp b/src/slave/containerizer/mesos/provisioner/backends/bind.hpp index 9eda944..2b2fcdc 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/bind.hpp +++ b/src/slave/containerizer/mesos/provisioner/backends/bind.hpp @@ -53,7 +53,8 @@ public: virtual process::Future<Nothing> provision( const std::vector<std::string>& layers, - const std::string& rootfs); + const std::string& rootfs, + const std::string& backendDir); virtual process::Future<bool> destroy(const std::string& rootfs); http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backends/copy.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp b/src/slave/containerizer/mesos/provisioner/backends/copy.cpp index f353c89..b9f6d7a 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/copy.cpp +++ b/src/slave/containerizer/mesos/provisioner/backends/copy.cpp @@ -77,7 +77,8 @@ CopyBackend::CopyBackend(Owned<CopyBackendProcess> _process) Future<Nothing> CopyBackend::provision( const vector<string>& layers, - const string& rootfs) + const string& rootfs, + const string& backendDir) { return dispatch( process.get(), &CopyBackendProcess::provision, layers, rootfs); http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backends/copy.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/backends/copy.hpp b/src/slave/containerizer/mesos/provisioner/backends/copy.hpp index b62507f..5bd6a52 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/copy.hpp +++ b/src/slave/containerizer/mesos/provisioner/backends/copy.hpp @@ -47,7 +47,8 @@ public: // path. virtual process::Future<Nothing> provision( const std::vector<std::string>& layers, - const std::string& rootfs); + const std::string& rootfs, + const std::string& backendDir); virtual process::Future<bool> destroy(const std::string& rootfs); http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp b/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp index 93892a7..ebe1fc5 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp +++ b/src/slave/containerizer/mesos/provisioner/backends/overlay.cpp @@ -45,7 +45,10 @@ namespace slave { class OverlayBackendProcess : public Process<OverlayBackendProcess> { public: - Future<Nothing> provision(const vector<string>& layers, const string& rootfs); + Future<Nothing> provision( + const vector<string>& layers, + const string& rootfs, + const string& backendDir); Future<bool> destroy(const string& rootfs); }; @@ -55,15 +58,15 @@ Try<Owned<Backend>> OverlayBackend::create(const Flags&) { Result<string> user = os::user(); if (!user.isSome()) { - return Error("Failed to determine user: " + - (user.isError() ? user.error() : "username not found")); + return Error( + "Failed to determine user: " + + (user.isError() ? user.error() : "username not found")); } if (user.get() != "root") { return Error( "OverlayBackend requires root privileges, " - "but is running as user " + - user.get()); + "but is running as user " + user.get()); } Try<bool> supported = fs::overlay::supported(); @@ -96,13 +99,17 @@ OverlayBackend::OverlayBackend(Owned<OverlayBackendProcess> _process) Future<Nothing> OverlayBackend::provision( const vector<string>& layers, - const string& rootfs) + const string& rootfs, + const string& backendDir) { return dispatch( - process.get(), &OverlayBackendProcess::provision, layers, rootfs); + process.get(), + &OverlayBackendProcess::provision, + layers, + rootfs, + backendDir); } - Future<bool> OverlayBackend::destroy(const string& rootfs) { return dispatch(process.get(), &OverlayBackendProcess::destroy, rootfs); @@ -111,34 +118,54 @@ Future<bool> OverlayBackend::destroy(const string& rootfs) Future<Nothing> OverlayBackendProcess::provision( const vector<string>& layers, - const string& rootfs) + const string& rootfs, + const string& backendDir) { if (layers.size() == 0) { return Failure("No filesystem layer provided"); } - if (layers.size() == 1) { - return Failure("Need more than one layer for overlay"); + Try<Nothing> mkdir = os::mkdir(rootfs); + if (mkdir.isError()) { + return Failure( + "Failed to create container rootfs at '" + + rootfs + "': " + mkdir.error()); } - Try<Nothing> mkdir = os::mkdir(rootfs); + const string scratchDirId = Path(rootfs).basename(); + const string scratchDir = path::join(backendDir, "scratch", scratchDirId); + const string upperdir = path::join(scratchDir, "upperdir"); + const string workdir = path::join(scratchDir, "workdir"); + + mkdir = os::mkdir(upperdir); if (mkdir.isError()) { - return Failure("Failed to create container rootfs at '" + rootfs + "': " + - mkdir.error()); + return Failure( + "Failed to create overlay upperdir at '" + + upperdir + "': " + mkdir.error()); } - // For overlayfs, the specified lower directories will be stacked beginning - // from the rightmost one and going left. But we need the first layer in the - // vector to be the the bottom most layer. - std::string lowerDir = "lowerdir="; - lowerDir += strings::join(":", adaptor::reverse(layers)); + mkdir = os::mkdir(workdir); + if (mkdir.isError()) { + return Failure( + "Failed to create overlay workdir at '" + + workdir + "': " + mkdir.error()); + } + + // For overlayfs, the specified lower directories will be stacked + // beginning from the rightmost one and going left. But we need the + // first layer in the vector to be the the bottom most layer. + string options = "lowerdir=" + strings::join(":", adaptor::reverse(layers)); + options += ",upperdir=" + upperdir; + options += ",workdir=" + workdir; + + VLOG(1) << "Provisioning image rootfs with overlayfs: '" << options << "'"; Try<Nothing> mount = fs::mount( "overlay", rootfs, "overlay", - MS_RDONLY, - lowerDir); + 0, + options); if (mount.isError()) { return Failure( http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp b/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp index 85cc737..387f28a 100644 --- a/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp +++ b/src/slave/containerizer/mesos/provisioner/backends/overlay.hpp @@ -27,10 +27,20 @@ namespace slave { class OverlayBackendProcess; -// This is a specialized backend that is useful for deploying multi-layer -// images using the overlayfs-based backend. -// 1) OverlayBackend does not support images with a single layer. -// 2) The filesystem is read-only. +// This backend mounts the images layers to the rootfs using the overlay file +// system. The directory layout is as follows: +// <work_dir> ('--work_dir' flag) +// |-- provisioner +// |-- containers +// |-- <container-id> +// |-- backends +// |-- overlay +// |-- rootfses +// |-- <rootfs_id> (the rootfs) +// |-- scratch +// |-- <rootfs_id> (the scratch space) +// |-- upperdir +// |-- workdir class OverlayBackend : public Backend { public: @@ -40,7 +50,8 @@ public: virtual process::Future<Nothing> provision( const std::vector<std::string>& layers, - const std::string& rootfs); + const std::string& rootfs, + const std::string& backendDir); virtual process::Future<bool> destroy(const std::string& rootfs); http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/paths.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/paths.cpp b/src/slave/containerizer/mesos/provisioner/paths.cpp index 07581f6..86a45f3 100644 --- a/src/slave/containerizer/mesos/provisioner/paths.cpp +++ b/src/slave/containerizer/mesos/provisioner/paths.cpp @@ -183,6 +183,19 @@ Try<hashmap<string, hashset<string>>> listContainerRootfses( return results; } + +string getBackendDir( + const string& provisionerDir, + const ContainerID& containerId, + const string& backend) +{ + return getBackendDir( + getBackendsDir( + getContainerDir( + provisionerDir, containerId)), + backend); +} + } // namespace paths { } // namespace provisioner { } // namespace slave { http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/paths.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/paths.hpp b/src/slave/containerizer/mesos/provisioner/paths.hpp index 2ea38ac..9829d6b 100644 --- a/src/slave/containerizer/mesos/provisioner/paths.hpp +++ b/src/slave/containerizer/mesos/provisioner/paths.hpp @@ -45,6 +45,7 @@ namespace paths { // Under each backend a rootfs is identified by the 'rootfs_id' which // is a UUID. + std::string getContainerDir( const std::string& provisionerDir, const ContainerID& containerId); @@ -69,6 +70,12 @@ listContainerRootfses( Try<hashset<ContainerID>> listContainers( const std::string& provisionerDir); + +std::string getBackendDir( + const std::string& provisionerDir, + const ContainerID& containerId, + const std::string& backend); + } // namespace paths { } // namespace provisioner { } // namespace slave { http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/slave/containerizer/mesos/provisioner/provisioner.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/provisioner.cpp b/src/slave/containerizer/mesos/provisioner/provisioner.cpp index 8a4938e..dcbbbaf 100644 --- a/src/slave/containerizer/mesos/provisioner/provisioner.cpp +++ b/src/slave/containerizer/mesos/provisioner/provisioner.cpp @@ -293,7 +293,15 @@ Future<ProvisionInfo> ProvisionerProcess::_provision( infos[containerId]->rootfses[backend].insert(rootfsId); - return backends.get(backend).get()->provision(ImageInfo.layers, rootfs) + string backendDir = provisioner::paths::getBackendDir( + rootDir, + containerId, + backend); + + return backends.get(backend).get()->provision( + ImageInfo.layers, + rootfs, + backendDir) .then([rootfs, ImageInfo]() -> Future<ProvisionInfo> { return ProvisionInfo{rootfs, ImageInfo.dockerManifest}; }); http://git-wip-us.apache.org/repos/asf/mesos/blob/4dfa91fc/src/tests/containerizer/provisioner_backend_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/containerizer/provisioner_backend_tests.cpp b/src/tests/containerizer/provisioner_backend_tests.cpp index d49204f..bc04760 100644 --- a/src/tests/containerizer/provisioner_backend_tests.cpp +++ b/src/tests/containerizer/provisioner_backend_tests.cpp @@ -91,20 +91,27 @@ TEST_F(OverlayBackendTest, ROOT_OVERLAYFS_OverlayFSBackend) hashmap<string, Owned<Backend>> backends = Backend::create(slave::Flags()); ASSERT_TRUE(backends.contains("overlay")); - AWAIT_READY(backends["overlay"]->provision({layer1, layer2}, rootfs)); + AWAIT_READY(backends["overlay"]->provision( + {layer1, layer2}, + rootfs, + sandbox.get())); EXPECT_TRUE(os::exists(path::join(rootfs, "dir1", "1"))); - Try<string> read = os::read(path::join(rootfs, "dir1", "1")); - EXPECT_SOME_EQ("1", read); + EXPECT_SOME_EQ("1", os::read(path::join(rootfs, "dir1", "1"))); EXPECT_TRUE(os::exists(path::join(rootfs, "dir2", "2"))); - read = os::read(path::join(rootfs, "dir2", "2")); - EXPECT_SOME_EQ("2", read); + EXPECT_SOME_EQ("2", os::read(path::join(rootfs, "dir2", "2"))); - EXPECT_TRUE(os::exists(path::join(rootfs, "file"))); - read = os::read(path::join(rootfs, "file")); // Last layer should overwrite existing file of earlier layers. - EXPECT_SOME_EQ("test2", read); + EXPECT_TRUE(os::exists(path::join(rootfs, "file"))); + EXPECT_SOME_EQ("test2", os::read(path::join(rootfs, "file"))); + + // Rootfs should be writable. + ASSERT_SOME(os::write(path::join(rootfs, "file"), "test3")); + + // Files created in rootfs should shadow the files of lower dirs. + EXPECT_SOME_EQ("test3", os::read(path::join(rootfs, "file"))); + EXPECT_SOME_EQ("test2", os::read(path::join(layer2, "file"))); AWAIT_READY(backends["overlay"]->destroy(rootfs)); @@ -130,7 +137,10 @@ TEST_F(BindBackendTest, ROOT_BindBackend) string target = path::join(os::getcwd(), "target"); - AWAIT_READY(backends["bind"]->provision({rootfs}, target)); + AWAIT_READY(backends["bind"]->provision( + {rootfs}, + target, + sandbox.get())); EXPECT_TRUE(os::stat::isdir(path::join(target, "tmp"))); @@ -170,24 +180,20 @@ TEST_F(CopyBackendTest, ROOT_CopyBackend) hashmap<string, Owned<Backend>> backends = Backend::create(slave::Flags()); ASSERT_TRUE(backends.contains("copy")); - AWAIT_READY(backends["copy"]->provision({layer1, layer2}, rootfs)); + AWAIT_READY(backends["copy"]->provision( + {layer1, layer2}, + rootfs, + sandbox.get())); EXPECT_TRUE(os::exists(path::join(rootfs, "dir1", "1"))); - Try<string> read = os::read(path::join(rootfs, "dir1", "1")); - ASSERT_SOME(read); - EXPECT_EQ("1", read.get()); + EXPECT_SOME_EQ("1", os::read(path::join(rootfs, "dir1", "1"))); EXPECT_TRUE(os::exists(path::join(rootfs, "dir2", "2"))); - read = os::read(path::join(rootfs, "dir2", "2")); - ASSERT_SOME(read); - EXPECT_EQ("2", read.get()); - - EXPECT_TRUE(os::exists(path::join(rootfs, "file"))); - read = os::read(path::join(rootfs, "file")); - ASSERT_SOME(read); + EXPECT_SOME_EQ("2", os::read(path::join(rootfs, "dir2", "2"))); // Last layer should overwrite existing file. - EXPECT_EQ("test2", read.get()); + EXPECT_TRUE(os::exists(path::join(rootfs, "file"))); + EXPECT_SOME_EQ("test2", os::read(path::join(rootfs, "file"))); AWAIT_READY(backends["copy"]->destroy(rootfs));