Add Docker local store recover.
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/fb2df960 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/fb2df960 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/fb2df960 Branch: refs/heads/master Commit: fb2df960a805132e58624ee0d73eaa098d57d67a Parents: 11dc584 Author: Timothy Chen <[email protected]> Authored: Mon Sep 7 23:07:55 2015 -0700 Committer: Timothy Chen <[email protected]> Committed: Fri Sep 25 09:02:05 2015 -0700 ---------------------------------------------------------------------- src/Makefile.am | 8 +- src/slave/containerizer/provisioners/docker.cpp | 109 ++++++---------- src/slave/containerizer/provisioners/docker.hpp | 1 - .../provisioners/docker/local_store.cpp | 123 ++++++++++--------- .../provisioners/docker/local_store.hpp | 21 ++-- .../provisioners/docker/reference_store.cpp | 28 ++--- .../provisioners/docker/reference_store.hpp | 11 +- .../containerizer/provisioners/docker/store.hpp | 17 ++- 8 files changed, 147 insertions(+), 171 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/fb2df960/src/Makefile.am ---------------------------------------------------------------------- diff --git a/src/Makefile.am b/src/Makefile.am index 65def70..cd8b2ca 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -529,6 +529,10 @@ libmesos_no_3rdparty_la_SOURCES = \ slave/containerizer/provisioner/appc/store.cpp \ slave/containerizer/provisioner/backend.cpp \ slave/containerizer/provisioner/backends/copy.cpp \ + slave/containerizer/provisioner/docker.cpp \ + slave/containerizer/provisioner/docker/local_store.cpp \ + slave/containerizer/provisioner/docker/paths.cpp \ + slave/containerizer/provisioner/docker/reference_store.cpp \ slave/containerizer/provisioner/docker/registry_client.cpp \ slave/containerizer/provisioner/docker/token_manager.cpp \ slave/resource_estimators/noop.cpp \ @@ -701,10 +705,6 @@ if OS_LINUX libmesos_no_3rdparty_la_SOURCES += slave/containerizer/isolators/filesystem/shared.cpp libmesos_no_3rdparty_la_SOURCES += slave/containerizer/linux_launcher.cpp libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/backends/bind.cpp - libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/docker.cpp - libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/docker/local_store.cpp - libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/docker/paths.cpp - libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/docker/reference_store.cpp else EXTRA_DIST += linux/cgroups.cpp EXTRA_DIST += linux/fs.cpp http://git-wip-us.apache.org/repos/asf/mesos/blob/fb2df960/src/slave/containerizer/provisioners/docker.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioners/docker.cpp b/src/slave/containerizer/provisioners/docker.cpp index 32e1a3b..b1f737f 100644 --- a/src/slave/containerizer/provisioners/docker.cpp +++ b/src/slave/containerizer/provisioners/docker.cpp @@ -30,10 +30,14 @@ #include <process/owned.hpp> #include <process/sequence.hpp> -#include "linux/fs.hpp" +#include "slave/containerizer/provisioners/backend.hpp" +#include "slave/containerizer/provisioners/paths.hpp" +#include "slave/containerizer/provisioners/docker/paths.hpp" #include "slave/containerizer/provisioners/docker/store.hpp" +#include "slave/paths.hpp" + using namespace process; using namespace mesos::internal::slave; @@ -68,20 +72,18 @@ public: private: DockerProvisionerProcess( - const Flags& flags, - const process::Owned<Store>& store, - const process::Owned<mesos::internal::slave::Backend>& backend); + const string& _rootDir, + const Flags& _flags, + const process::Owned<Store>& _store, + const hashmap<std::string, process::Owned<Backend>>& _backends); process::Future<std::string> _provision( + const DockerImage& image, const ContainerID& containerId, - const DockerImage& image); - - process::Future<DockerImage> fetch( - const std::string& name, - const std::string& sandbox); + const string& rootfs); + const string& rootDir; const Flags flags; - process::Owned<Store> store; hashmap<string, process::Owned<Backend>> backends; @@ -92,7 +94,6 @@ private: }; hashmap<ContainerID, Owned<Info>> infos; - }; @@ -179,24 +180,17 @@ Try<Owned<DockerProvisionerProcess>> DockerProvisionerProcess::create( const Flags& flags, Fetcher* fetcher) { - string _root = + string rootDir = slave::paths::getProvisionerDir(flags.work_dir, Image::DOCKER); - Try<Nothing> mkdir = os::mkdir(_root); - if (mkdir.isError()) { - return Error("Failed to create provisioner root directory '" + - _root + "': " + mkdir.error()); - } - - Result<string> root = os::realpath(_root); - if (root.isError()) { - return Error( - "Failed to resolve the realpath of provisioner root directory '" + - _root + "': " + root.error()); + if (!os::exists(rootDir)) { + Try<Nothing> mkdir = os::mkdir(rootDir); + if (mkdir.isError()) { + return Error("Failed to create Docker provisioner root directory '" + + rootDir + "': " + mkdir.error()); + } } - CHECK_SOME(root); // Can't be None since we just created it. - hashmap<string, Owned<Backend>> backends = Backend::create(flags); if (backends.empty()) { return Error("No usable Docker provisioner backend created"); @@ -214,6 +208,7 @@ Try<Owned<DockerProvisionerProcess>> DockerProvisionerProcess::create( return Owned<DockerProvisionerProcess>( new DockerProvisionerProcess( + rootDir, flags, store.get(), backends)); @@ -221,10 +216,12 @@ Try<Owned<DockerProvisionerProcess>> DockerProvisionerProcess::create( DockerProvisionerProcess::DockerProvisionerProcess( + const string& _rootDir, const Flags& _flags, const Owned<Store>& _store, const hashmap<string, Owned<Backend>>& _backends) - : flags(_flags), + : rootDir(_rootDir), + flags(_flags), store(_store), backends(_backends) {} @@ -252,7 +249,7 @@ Future<Nothing> DockerProvisionerProcess::recover( // be destroyed by the containerizer using the normal cleanup path. See // MESOS-2367 for details. Try<hashmap<ContainerID, string>> containers = - provisioners::paths::listContainers(root); + provisioners::paths::listContainers(rootDir); if (containers.isError()) { return Failure("Failed to list the containers managed by Docker " @@ -265,7 +262,7 @@ Future<Nothing> DockerProvisionerProcess::recover( Owned<Info> info = Owned<Info>(new Info()); Try<hashmap<string, hashmap<string, string>>> rootfses = - provisioners::paths::listContainerRootfses(root, containerId); + provisioners::paths::listContainerRootfses(rootDir, containerId); if (rootfses.isError()) { return Failure("Unable to list rootfses belonged to container '" + @@ -289,7 +286,7 @@ Future<Nothing> DockerProvisionerProcess::recover( // Destroy (unknown) orphan container's rootfses. Try<hashmap<string, hashmap<string, string>>> rootfses = - provisioners::paths::listContainerRootfses(root, containerId); + provisioners::paths::listContainerRootfses(rootDir, containerId); if (rootfses.isError()) { return Failure("Unable to find rootfses for container '" + @@ -340,7 +337,7 @@ Future<string> DockerProvisionerProcess::provision( string rootfsId = UUID::random().toString(); string rootfs = provisioners::paths::getContainerRootfsDir( - root, containerId, flags.docker_backend, rootfsId); + rootDir, containerId, flags.docker_backend, rootfsId); if (!infos.contains(containerId)) { infos.put(containerId, Owned<Info>(new Info())); @@ -348,63 +345,29 @@ Future<string> DockerProvisionerProcess::provision( infos[containerId]->rootfses[flags.docker_backend].put(rootfsId, rootfs); - - return fetch(image.docker().name()) - .then(defer(self(), - &Self::_provision, - containerId, - lambda::_1)); + return store->get(image.docker().name()) + .then(defer(self(), &Self::_provision, lambda::_1, containerId, rootfs)); } Future<string> DockerProvisionerProcess::_provision( + const DockerImage& image, const ContainerID& containerId, - const DockerImage& image) + const string& rootfs) { CHECK(backends.contains(flags.docker_backend)); - // Create root directory. - string base = path::join(flags.docker_rootfs_dir, - stringify(containerId)); - - string rootfs = path::join(base, "rootfs"); - - Try<Nothing> mkdir = os::mkdir(base); - if (mkdir.isError()) { - return Failure("Failed to create directory for container filesystem: " + - mkdir.error()); - } - LOG(INFO) << "Provisioning rootfs for container '" << containerId << "'" - << " to '" << base << "'"; + << " to '" << rootfs << "'"; vector<string> layerPaths; foreach (const string& layerId, image.layers) { - layerPaths.push_back(path::join(flags.docker_store_dir, layerId, "rootfs")); + layerPaths.push_back( + paths::getImageLayerRootfsPath(flags.docker_store_dir, layerId)); } - - return backends[flags.docker_backend]->provision(layerPaths, base) - .then([rootfs]() -> Future<string> { - // Bind mount the rootfs to itself so we can pivot_root. We do - // it now so any subsequent mounts by the containerizer or - // isolators are correctly handled by pivot_root. - Try<Nothing> mount = - fs::mount(rootfs, rootfs, None(), MS_BIND | MS_SHARED, NULL); - if (mount.isError()) { - return Failure("Failure to bind mount rootfs: " + mount.error()); - } - - return rootfs; - }); -} - - -// Fetch an image and all dependencies. -Future<DockerImage> DockerProvisionerProcess::fetch( - const string& name) -{ - return store->get(name); + return backends[flags.docker_backend]->provision(layerPaths, rootfs) + .then([rootfs]() -> Future<string> { return rootfs; }); } http://git-wip-us.apache.org/repos/asf/mesos/blob/fb2df960/src/slave/containerizer/provisioners/docker.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioners/docker.hpp b/src/slave/containerizer/provisioners/docker.hpp index 850ce85..35e23c9 100644 --- a/src/slave/containerizer/provisioners/docker.hpp +++ b/src/slave/containerizer/provisioners/docker.hpp @@ -41,7 +41,6 @@ #include <mesos/resources.hpp> #include "slave/containerizer/provisioner.hpp" -#include "slave/containerizer/provisioners/backend.hpp" #include "slave/flags.hpp" http://git-wip-us.apache.org/repos/asf/mesos/blob/fb2df960/src/slave/containerizer/provisioners/docker/local_store.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioners/docker/local_store.cpp b/src/slave/containerizer/provisioners/docker/local_store.cpp index aec7df9..c6a9efe 100644 --- a/src/slave/containerizer/provisioners/docker/local_store.cpp +++ b/src/slave/containerizer/provisioners/docker/local_store.cpp @@ -62,10 +62,13 @@ public: process::Future<DockerImage> get(const std::string& name); + process::Future<Nothing> recover(); + private: LocalStoreProcess( - const Flags& flags, - Owned<ReferenceStore> _refStore); + const Flags& _flags, + Owned<ReferenceStore> _refStore) + : flags(_flags), refStore(_refStore) {} process::Future<Nothing> untarImage( const std::string& tarPath, @@ -73,7 +76,7 @@ private: process::Future<DockerImage> putImage( const std::string& name, - const std::string& staging) + const std::string& staging); Result<std::string> getParentId( const std::string& staging, @@ -81,18 +84,17 @@ private: process::Future<Nothing> putLayers( const std::string& staging, - const std::list<std::string>& layers) + const std::list<std::string>& layers); process::Future<Nothing> putLayer( const std::string& staging, - const std::string& id) + const std::string& id); process::Future<Nothing> moveLayer( const std::string& staging, - const std::string& id) + const std::string& id); const Flags flags; - process::Owned<ReferenceStore> refStore; }; @@ -117,42 +119,17 @@ Try<Owned<Store>> LocalStore::create( const Flags& flags, Fetcher* fetcher) { - if (!os::exists(flags.docker_store_dir)) { - Try<Nothing> mkdir = os::mkdir(flags.docker_store_dir); - if (mkdir.isError()) { - return Error("Failed to create Docker store directory: " + mkdir.error()); - } - } - - if (!os::exists(paths::getStagingDir(flags.docker_store_dir))) { - Try<Nothing> mkdir = - os::mkdir(paths::getStagingDir(flags.docker_store_dir)); - if (mkdir.isError()) { - return Error("Failed to create Docker store staging directory: " + - mkdir.error()); - } - } - Try<Owned<LocalStoreProcess>> process = LocalStoreProcess::create(flags, fetcher); if (process.isError()) { return Error(process.error()); } - Try<Owned<ReferenceStore>> refStore = ReferenceStore::create(flags); - if (refStore.isError()) { - return Error(refStore); - } - - return Owned<Store>(new LocalStore(process.get(), refStore.get())); + return Owned<Store>(new LocalStore(process.get())); } -LocalStore::LocalStore( - Owned<LocalStoreProcess> process, - Owned<ReferenceStore> refStore) - : process(process), - _refStore(refStore) +LocalStore::LocalStore(Owned<LocalStoreProcess> _process) : process(_process) { process::spawn(CHECK_NOTNULL(process.get())); } @@ -165,49 +142,81 @@ LocalStore::~LocalStore() } -Future<Option<DockerImage>> LocalStore::get(const string& name) +Future<DockerImage> LocalStore::get(const string& name) { return dispatch(process.get(), &LocalStoreProcess::get, name); } +Future<Nothing> LocalStore::recover() +{ + return dispatch(process.get(), &LocalStoreProcess::recover); +} + + Try<Owned<LocalStoreProcess>> LocalStoreProcess::create( const Flags& flags, Fetcher* fetcher) { - return Owned<LocalStoreProcess>(new LocalStoreProcess(flags)); -} + if (!os::exists(flags.docker_store_dir)) { + Try<Nothing> mkdir = os::mkdir(flags.docker_store_dir); + if (mkdir.isError()) { + return Error("Failed to create Docker store directory: " + mkdir.error()); + } + } + + if (!os::exists(paths::getStagingDir(flags.docker_store_dir))) { + Try<Nothing> mkdir = + os::mkdir(paths::getStagingDir(flags.docker_store_dir)); + if (mkdir.isError()) { + return Error("Failed to create Docker store staging directory: " + + mkdir.error()); + } + } + Try<Owned<ReferenceStore>> refStore = ReferenceStore::create(flags); + if (refStore.isError()) { + return Error(refStore.error()); + } -LocalStoreProcess::LocalStoreProcess(const Flags& flags) - : flags(flags), refStore(ReferenceStore::create(flags).get()) {} + return Owned<LocalStoreProcess>(new LocalStoreProcess(flags, refStore.get())); +} Future<DockerImage> LocalStoreProcess::get(const string& name) { - Option<DockerImage> image = refStore->get(name); - if (image.isSome()) { - return image.get(); - } + return refStore->get(name) + .then(defer(self(), + [this, &name]( + const Option<DockerImage>& image) -> Future<DockerImage> { + if (image.isSome()) { + return image.get(); + } - string tarPath = - paths::getLocalImageTarPath(flags.docker_discovery_local_dir, name); - if (!os::exists(tarPath)) { - return Failure("No Docker image tar archive found"); - } + string tarPath = + paths::getLocalImageTarPath(flags.docker_discovery_local_dir, name); + if (!os::exists(tarPath)) { + return Failure("No Docker image tar archive found"); + } - // Create a temporary staging directory. - Try<string> staging = - os::mkdtemp(paths::getTempStaging(flags.docker_store_dir)); - if (staging.isError()) { - return Failure("Failed to create a staging directory"); - } + // Create a temporary staging directory. + Try<string> staging = + os::mkdtemp(paths::getTempStaging(flags.docker_store_dir)); + if (staging.isError()) { + return Failure("Failed to create a staging directory"); + } - return untarImage(tarPath, staging.get()) - .then(defer(self(), &Self::putImage, name, staging.get())); + return untarImage(tarPath, staging.get()) + .then(defer(self(), &Self::putImage, name, staging.get())); + })); } +Future<Nothing> LocalStoreProcess::recover() +{ + return refStore->recover(); +} + Future<Nothing> LocalStoreProcess::untarImage( const string& tarPath, const string& staging) @@ -376,7 +385,7 @@ Future<Nothing> LocalStoreProcess::putLayer( const string imageLayerPath = paths::getImageLayerPath(flags.docker_store_dir, id); - if (!os::exists()) { + if (!os::exists(imageLayerPath)) { Try<Nothing> mkdir = os::mkdir(imageLayerPath); if (mkdir.isError()) { return Failure("Failed to create Image layer directory '" + http://git-wip-us.apache.org/repos/asf/mesos/blob/fb2df960/src/slave/containerizer/provisioners/docker/local_store.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioners/docker/local_store.hpp b/src/slave/containerizer/provisioners/docker/local_store.hpp index 2f0c9f1..b650b5e 100644 --- a/src/slave/containerizer/provisioners/docker/local_store.hpp +++ b/src/slave/containerizer/provisioners/docker/local_store.hpp @@ -26,8 +26,17 @@ namespace internal { namespace slave { namespace docker { +// Forward declaration. class LocalStoreProcess; +class ReferenceStore; + +/** + * LocalStore assumes Docker images are stored in a local directory + * (configured with flags.docker_discovery_local_dir), with all the + * images saved as tar with the name as the image name with tag (e.g: + * ubuntu:14.04.tar). + */ class LocalStore : public Store { public: @@ -37,18 +46,12 @@ public: const Flags& flags, Fetcher* fetcher); - /** - * Put assumes the image tar archive is located in the directory specified in - * the slave flag docker_discovery_local_dir and is named with <name>.tar . - */ - virtual process::Future<DockerImage> put( - const std::string& name, - const std::string& sandbox); + virtual process::Future<DockerImage> get(const std::string& name); - virtual process::Future<Option<DockerImage>> get(const std::string& name); + virtual process::Future<Nothing> recover(); private: - explicit LocalStore(process::Owned<LocalStoreProcess> process); + explicit LocalStore(process::Owned<LocalStoreProcess> _process); LocalStore(const LocalStore&); // Not copyable. http://git-wip-us.apache.org/repos/asf/mesos/blob/fb2df960/src/slave/containerizer/provisioners/docker/reference_store.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioners/docker/reference_store.cpp b/src/slave/containerizer/provisioners/docker/reference_store.cpp index 1567248..4b72319 100644 --- a/src/slave/containerizer/provisioners/docker/reference_store.cpp +++ b/src/slave/containerizer/provisioners/docker/reference_store.cpp @@ -54,18 +54,15 @@ class ReferenceStoreProcess : public process::Process<ReferenceStoreProcess> public: ~ReferenceStoreProcess() {} - // Explicitly use 'initialize' since we are overloading below. - using process::ProcessBase::initialize; - - void initialize(); - static Try<process::Owned<ReferenceStoreProcess>> create(const Flags& flags); - process::Future<DockerImage> put( + Future<DockerImage> put( const std::string& name, const std::list<std::string>& layers); - process::Future<Option<DockerImage>> get(const std::string& name); + Future<Option<DockerImage>> get(const std::string& name); + + Future<Nothing> recover(); // TODO(chenlily): Implement removal of unreferenced images. @@ -109,9 +106,9 @@ ReferenceStore::~ReferenceStore() } -void ReferenceStore::initialize() +Future<Nothing> ReferenceStore::recover() { - process::dispatch(process.get(), &ReferenceStoreProcess::initialize); + return process::dispatch(process.get(), &ReferenceStoreProcess::recover); } @@ -194,7 +191,7 @@ Try<Nothing> ReferenceStoreProcess::persist() } -void ReferenceStoreProcess::initialize() +Future<Nothing> ReferenceStoreProcess::recover() { string storedImagesPath = paths::getStoredImagesPath(flags.docker_store_dir); @@ -202,15 +199,14 @@ void ReferenceStoreProcess::initialize() if (!os::exists(storedImagesPath)) { LOG(INFO) << "No images to load from disk. Docker provisioner image " << "storage path: " << storedImagesPath << " does not exist."; - return; + return Nothing(); } Result<DockerProvisionerImages> images = ::protobuf::read<DockerProvisionerImages>(storedImagesPath); if (images.isError()) { - LOG(ERROR) << "Failed to read protobuf for Docker provisioner image: " - << images.error(); - return; + return Failure("Failed to read protobuf for Docker provisioner image: " + + images.error()); } for (int i = 0; i < images.get().images_size(); i++) { @@ -224,7 +220,7 @@ void ReferenceStoreProcess::initialize() layers.push_back(layerId); if (!os::exists( - paths::getImageLayerPath(flags.docker_store_dir, layerId))) { + paths::getImageLayerRootfsPath(flags.docker_store_dir, layerId))) { missingLayers.push_back(layerId); } } @@ -243,6 +239,8 @@ void ReferenceStoreProcess::initialize() } LOG(INFO) << "Loaded " << storedImages.size() << " Docker images."; + + return Nothing(); } } // namespace docker { http://git-wip-us.apache.org/repos/asf/mesos/blob/fb2df960/src/slave/containerizer/provisioners/docker/reference_store.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioners/docker/reference_store.hpp b/src/slave/containerizer/provisioners/docker/reference_store.hpp index 66b7573..be652ae 100644 --- a/src/slave/containerizer/provisioners/docker/reference_store.hpp +++ b/src/slave/containerizer/provisioners/docker/reference_store.hpp @@ -56,12 +56,6 @@ class ReferenceStore public: ~ReferenceStore(); - /** - * Recover all Docker Images that are on disk by checking if all - * layer dependencies for that layer are present on disk. - */ - void initialize(); - static Try<process::Owned<ReferenceStore>> create(const Flags& flags); /** @@ -85,6 +79,11 @@ public: */ process::Future<Option<DockerImage>> get(const std::string& name); + /** + * Recover all stored DockerImage and its layer references. + */ + process::Future<Nothing> recover(); + private: explicit ReferenceStore(process::Owned<ReferenceStoreProcess> process); http://git-wip-us.apache.org/repos/asf/mesos/blob/fb2df960/src/slave/containerizer/provisioners/docker/store.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioners/docker/store.hpp b/src/slave/containerizer/provisioners/docker/store.hpp index b9cb770..a9201d5 100644 --- a/src/slave/containerizer/provisioners/docker/store.hpp +++ b/src/slave/containerizer/provisioners/docker/store.hpp @@ -49,14 +49,19 @@ public: virtual ~Store() {} /** - * Get image by name. - * - * @param name The name of the Docker image to retrieve from store. - * - * @return The DockerImage that holds the Docker layers. - */ + * Get image by name. + * + * @param name The name of the Docker image to retrieve from store. + * + * @return The DockerImage that holds the Docker layers. + */ virtual process::Future<DockerImage> get(const std::string& name) = 0; + /** + * Recover all stored images + */ + virtual process::Future<Nothing> recover() = 0; + // TODO(chenlily): Implement removing an image. protected:
