Docker image store. Stored images currently kept indefinitely.
Review: https://reviews.apache.org/r/37197 Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/38319bfb Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/38319bfb Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/38319bfb Branch: refs/heads/master Commit: 38319bfb332138a0fea31944220a024c5e2ff1d1 Parents: 15c0d80 Author: Lily Chen <[email protected]> Authored: Mon Jul 6 15:39:55 2015 -0700 Committer: Timothy Chen <[email protected]> Committed: Fri Sep 25 09:02:04 2015 -0700 ---------------------------------------------------------------------- src/Makefile.am | 2 + .../containerizer/provisioners/docker/store.cpp | 441 +++++++++++++++++++ .../containerizer/provisioners/docker/store.hpp | 154 +++++++ src/slave/flags.cpp | 5 + src/slave/flags.hpp | 1 + 5 files changed, 603 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/38319bfb/src/Makefile.am ---------------------------------------------------------------------- diff --git a/src/Makefile.am b/src/Makefile.am index f082484..da72818 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -692,6 +692,7 @@ if OS_LINUX libmesos_no_3rdparty_la_SOURCES += slave/containerizer/isolators/filesystem/shared.cpp libmesos_no_3rdparty_la_SOURCES += slave/containerizer/linux_launcher.cpp libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/backends/bind.cpp + libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/docker/store.cpp else EXTRA_DIST += linux/cgroups.cpp EXTRA_DIST += linux/fs.cpp @@ -816,6 +817,7 @@ libmesos_no_3rdparty_la_SOURCES += \ slave/containerizer/provisioner/backends/bind.hpp \ slave/containerizer/provisioner/backends/copy.hpp \ slave/containerizer/provisioner/docker/registry_client.hpp \ + slave/containerizer/provisioner/docker/store.hpp \ slave/containerizer/provisioner/docker/token_manager.hpp \ slave/containerizer/isolators/posix.hpp \ slave/containerizer/isolators/posix/disk.hpp \ http://git-wip-us.apache.org/repos/asf/mesos/blob/38319bfb/src/slave/containerizer/provisioners/docker/store.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioners/docker/store.cpp b/src/slave/containerizer/provisioners/docker/store.cpp new file mode 100644 index 0000000..e33b570 --- /dev/null +++ b/src/slave/containerizer/provisioners/docker/store.cpp @@ -0,0 +1,441 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "slave/containerizer/provisioners/docker/store.hpp" + +#include <list> +#include <utility> + +#include <glog/logging.h> + +#include <stout/os.hpp> +#include <stout/json.hpp> + +#include <process/defer.hpp> +#include <process/dispatch.hpp> + +#include "common/status_utils.hpp" + +#include "slave/containerizer/fetcher.hpp" +#include "slave/flags.hpp" + +using namespace process; + +using std::list; +using std::string; +using std::vector; +using std::pair; + +namespace mesos { +namespace internal { +namespace slave { +namespace docker { + +Try<Owned<Store>> Store::create( + const Flags& flags, + Fetcher* fetcher) +{ + hashmap<string, Try<Owned<Store>>(*)(const Flags&, Fetcher*)> creators{ + {"local", &LocalStore::create} + }; + + if (!creators.contains(flags.docker_store)) { + return Error("Unknown Docker store: " + flags.docker_store); + } + + return creators[flags.docker_store](flags, fetcher); +} + + +Try<Owned<Store>> LocalStore::create( + const Flags& flags, + Fetcher* fetcher) +{ + Try<Owned<LocalStoreProcess>> process = + LocalStoreProcess::create(flags, fetcher); + if (process.isError()) { + return Error("Failed to create store: " + process.error()); + } + + return Owned<Store>(new LocalStore(process.get())); +} + + +LocalStore::LocalStore(Owned<LocalStoreProcess> process) + : process(process) +{ + process::spawn(CHECK_NOTNULL(process.get())); +} + + +LocalStore::~LocalStore() +{ + process::terminate(process.get()); + process::wait(process.get()); +} + + +Future<DockerImage> LocalStore::put( + const string& uri, + const string& name, + const string& directory) +{ + return dispatch(process.get(), &LocalStoreProcess::put, uri, name, directory); +} + + +Future<Option<DockerImage>> LocalStore::get(const string& name) +{ + return dispatch(process.get(), &LocalStoreProcess::get, name); +} + + +Try<Owned<LocalStoreProcess>> LocalStoreProcess::create( + const Flags& flags, + Fetcher* fetcher) +{ + Owned<LocalStoreProcess> store = + Owned<LocalStoreProcess>(new LocalStoreProcess(flags, fetcher)); + + Try<Nothing> restore = store->restore(); + if (restore.isError()) { + return Error("Failed to restore store: " + restore.error()); + } + + return store; +} + + +LocalStoreProcess::LocalStoreProcess( + const Flags& flags, + Fetcher* fetcher) + : flags(flags), + fetcher(fetcher) {} + +// Currently only local file:// uri supported. +// TODO(chenlily): Add support for fetching image from external uri. +Future<DockerImage> LocalStoreProcess::put( + const string& uri, + const string& name, + const string& directory) +{ + string imageUri = uri; + if (strings::startsWith(imageUri, "file://")) { + imageUri = imageUri.substr(7); + } + + Try<bool> isDir = os::stat::isdir(imageUri); + if (isDir.isError()) { + return Failure("Failed to check directory for uri '" +imageUri + "':" + + isDir.error()); + } else if (!isDir.get()) { + return Failure("Docker image uri '" + imageUri + "' is not a directory"); + } + + Try<string> repoPath = path::join(imageUri, "repositories"); + if (repoPath.isError()) { + return Failure("Failed to create path to repository: " + repoPath.error()); + } + + Try<string> value = os::read(repoPath.get()); + if (value.isError()) { + return Failure("Failed to read repository JSON: " + value.error()); + } + + Try<JSON::Object> json = JSON::parse<JSON::Object>(value.get()); + if (json.isError()) { + return Failure("Failed to parse JSON: " + json.error()); + } + + Try<pair<string, string>> repoTag = DockerImage::parseTag(name); + if (repoTag.isError()) { + return Failure("Failed to parse Docker image name: " + repoTag.error()); + } + + string repository = repoTag.get().first; + string tag = repoTag.get().second; + + Result<JSON::Object> repositoryValue = + json.get().find<JSON::Object>(repository); + if (repositoryValue.isError()) { + return Failure("Failed to find repository: " + repositoryValue.error()); + } else if (repositoryValue.isNone()) { + return Failure("Repository '" + repository + "' is not found"); + } + + JSON::Object repositoryJson = repositoryValue.get(); + + // We don't use JSON find here because a tag might contain a '.'. + std::map<string, JSON::Value>::const_iterator entry = + repositoryJson.values.find(tag); + if (entry == repositoryJson.values.end()) { + return Failure("Tag '" + tag + "' is not found"); + } else if (!entry->second.is<JSON::String>()) { + return Failure("Tag JSON value expected to be JSON::String"); + } + + Try<string> layerUri = path::join( + imageUri, + entry->second.as<JSON::String>().value); + if (layerUri.isError()) { + return Failure("Failed to create path to image layer: " + layerUri.error()); + } + + return putLayer(layerUri.get(), directory) + .then([=](const Shared<DockerLayer>& layer) -> Future<DockerImage> { + DockerImage image(name, layer); + images[name] = image; + return image; + }); +} + + +Future<Shared<DockerLayer>> LocalStoreProcess::putLayer( + const string& uri, + const string& directory) +{ + Try<string> hash = os::basename(uri); + if (hash.isError()) { + return Failure("Failed to determine hash for stored layer: " + + hash.error()); + } + + if (layers.contains(hash.get())) { + return layers[hash.get()]; + } + + return untarLayer(uri) + .then([=]() { + return entry(uri, directory); + }) + .then([=](const Shared<DockerLayer>& layer) { + VLOG(1) << "Stored layer with hash: " << hash.get(); + layers[hash.get()] = layer; + + return layer; + }); +} + + +Future<Nothing> LocalStoreProcess::untarLayer( + const string& uri) +{ + string rootFs = path::join(uri, "rootfs"); + + if (os::exists(rootFs)) { + return Nothing(); + } else { + os::mkdir(rootFs); + } + + // Untar imageUri/hash/layer.tar into imageUri/hash/rootfs. + vector<string> argv = { + "tar", + "-C", + rootFs, + "-x", + "-f", + path::join(uri, "layer.tar")}; + + Try<Subprocess> s = subprocess( + "tar", + argv, + Subprocess::PATH("/dev/null"), + Subprocess::PATH("/dev/null"), + Subprocess::PATH("/dev/null")); + if (s.isError()) { + return Failure("Failed to create tar subprocess: " + s.error()); + } + + return s.get().status() + .then([=](const Option<int>& status) -> Future<Nothing> { + if (status.isNone()) { + return Failure("Failed to reap status for tar subprocess in " + + uri); + } + + if (!WIFEXITED(status.get()) || WEXITSTATUS(status.get()) != 0) { + return Failure("Untar failed with exit code: " + + WSTRINGIFY(status.get())); + } + + return Nothing(); + }); +} + + +Future<Shared<DockerLayer>> LocalStoreProcess::storeLayer( + const string& hash, + const string& uri, + const string& directory) +{ + string store = uri; + + // Only copy if the store directory doesn't exist. + Future<Option<int>> status; + if (os::exists(store)) { + LOG(INFO) << "Layer store '" << store << "' exists, skipping rename"; + status = 0; + } else { + Try<int> out = os::open( + path::join(directory, "stdout"), + O_WRONLY | O_CREAT | O_TRUNC | O_NONBLOCK | O_CLOEXEC, + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (out.isError()) { + return Failure("Failed to create 'stdout' file: " + out.error()); + } + + // Repeat for stderr. + Try<int> err = os::open( + path::join(directory, "stderr"), + O_WRONLY | O_CREAT | O_TRUNC | O_NONBLOCK | O_CLOEXEC, + S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH); + if (err.isError()) { + os::close(out.get()); + return Failure("Failed to create 'stderr' file: " + err.error()); + } + + vector<string> argv{ + "cp", + "--archive", + path::join(uri, "rootfs"), + store + }; + + VLOG(1) << "Copying image with command: " << strings::join(" ", argv); + + Try<Subprocess> s = subprocess( + "cp", + argv, + Subprocess::PATH("/dev/null"), + Subprocess::FD(out.get()), + Subprocess::FD(err.get())); + if (s.isError()) { + return Failure("Failed to create 'cp' subprocess: " + s.error()); + } + + status = s.get().status(); + } + + return status + .then([=](const Option<int>& status) -> Future<Shared<DockerLayer>> { + if (status.isNone()) { + return Failure("Failed to reap subprocess to copy image"); + } else if (!WIFEXITED(status.get()) || WEXITSTATUS(status.get()) != 0) { + return Failure("Copy image failed with exit code: " + + WSTRINGIFY(status.get())); + } + + return entry(uri, directory); + }) + .then([=](const Shared<DockerLayer>& layer) { + LOG(INFO) << "Stored layer with hash: " << hash; + layers[hash] = layer; + + return layer; + }); +} + + +Future<Shared<DockerLayer>> LocalStoreProcess::entry( + const string& uri, + const string& directory) +{ + Result<string> realpath = os::realpath(uri); + if (realpath.isError()) { + return Failure("Error in checking store path: " + realpath.error()); + } else if (realpath.isNone()) { + return Failure("Store path not found"); + } + + Try<string> hash = os::basename(realpath.get()); + if (hash.isError()) { + return Failure( + "Failed to determine hash for stored image: " + hash.error()); + } + + Try<string> version = os::read(path::join(uri, "VERSION")); + if (version.isError()) { + return Failure("Failed to determine version of JSON: " + version.error()); + } + + Try<string> manifest = os::read(path::join(uri, "json")); + if (manifest.isError()) { + return Failure("Failed to read manifest: " + manifest.error()); + } + + Try<JSON::Object> json = JSON::parse<JSON::Object>(manifest.get()); + if (json.isError()) { + return Failure("Failed to parse manifest: " + json.error()); + } + + Result<JSON::String> parentId = json.get().find<JSON::String>("parent"); + if (parentId.isNone()) { + return Shared<DockerLayer>(new DockerLayer( + hash.get(), + json.get(), + realpath.get(), + version.get(), + None())); + } else if (parentId.isError()) { + return Failure("Failed to read parent of layer: " + parentId.error()); + } + + Try<string> uriDir = os::dirname(uri); + if (uriDir.isError()) { + return Failure("Failed to obtain layer directory: " + uriDir.error()); + } + + Try<string> parentUri = path::join(uriDir.get(), parentId.get().value); + if (parentUri.isError()) { + return Failure("Failed to create parent layer uri: " + parentUri.error()); + } + + return putLayer(parentUri.get(), directory) + .then([=](const Shared<DockerLayer>& parent) -> Shared<DockerLayer> { + return Shared<DockerLayer> (new DockerLayer( + hash.get(), + json.get(), + uri, + version.get(), + parent)); + }); +} + + +Future<Option<DockerImage>> LocalStoreProcess::get(const string& name) +{ + if (!images.contains(name)) { + return None(); + } + + return images[name]; +} + + +// Recover stored image layers and update layers map. +// TODO(chenlily): Implement restore. +Try<Nothing> LocalStoreProcess::restore() +{ + return Nothing(); +} + +} // namespace docker { +} // namespace slave { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/38319bfb/src/slave/containerizer/provisioners/docker/store.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioners/docker/store.hpp b/src/slave/containerizer/provisioners/docker/store.hpp new file mode 100644 index 0000000..6dd1332 --- /dev/null +++ b/src/slave/containerizer/provisioners/docker/store.hpp @@ -0,0 +1,154 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MESOS_DOCKER_STORE__ +#define __MESOS_DOCKER_STORE__ + +#include <string> +#include <vector> + +#include <stout/hashmap.hpp> +#include <stout/nothing.hpp> +#include <stout/option.hpp> +#include <stout/result.hpp> +#include <stout/try.hpp> + +#include <process/future.hpp> +#include <process/owned.hpp> +#include <process/process.hpp> +#include <process/shared.hpp> + +#include "slave/containerizer/fetcher.hpp" +#include "slave/containerizer/provisioners/docker.hpp" +#include "slave/flags.hpp" + +namespace mesos { +namespace internal { +namespace slave { +namespace docker { + +// Store fetches the Docker images and stores them on disk. +class Store +{ +public: + static Try<process::Owned<Store>> create( + const Flags& flags, + Fetcher* fetcher); + + virtual ~Store() {} + + // Put an image into to the store. Returns the DockerImage containing + // the manifest, hash of the image, and the path to the extracted + // image. + virtual process::Future<DockerImage> put( + const std::string& uri, + const std::string& name, + const std::string& directory) = 0; + + // Get image by name. + virtual process::Future<Option<DockerImage>> get(const std::string& name) = 0; + +protected: + Store() {} +}; + + +// Forward declaration. +class LocalStoreProcess; + +class LocalStore : public Store +{ +public: + virtual ~LocalStore(); + + static Try<process::Owned<Store>> create( + const Flags& flags, + Fetcher* fetcher); + + virtual process::Future<DockerImage> put( + const std::string& uri, + const std::string& name, + const std::string& directory); + + virtual process::Future<Option<DockerImage>> get(const std::string& name); + +private: + explicit LocalStore(process::Owned<LocalStoreProcess> process); + + LocalStore(const LocalStore&); // Not copyable. + LocalStore& operator=(const LocalStore&); // Not assignable. + + process::Owned<LocalStoreProcess> process; +}; + + +class LocalStoreProcess : public process::Process<LocalStoreProcess> +{ +public: + ~LocalStoreProcess() {} + + static Try<process::Owned<LocalStoreProcess>> create( + const Flags& flags, + Fetcher* fetcher); + + process::Future<DockerImage> put( + const std::string& uri, + const std::string& name, + const std::string& directory); + + process::Future<Option<DockerImage>> get(const std::string& name); + +private: + LocalStoreProcess( + const Flags& flags, + Fetcher* fetcher); + + Try<Nothing> restore(); + + process::Future<process::Shared<DockerLayer>> putLayer( + const std::string& uri, + const std::string& directory); + + process::Future<Nothing> untarLayer( + const std::string& uri); + + process::Future<process::Shared<DockerLayer>> storeLayer( + const std::string& hash, + const std::string& uri, + const std::string& directory); + + process::Future<process::Shared<DockerLayer>> entry( + const std::string& uri, + const std::string& directory); + + const Flags flags; + + // name -> DockerImage + hashmap<std::string, DockerImage> images; + // hash -> DockerLayer + hashmap<std::string, process::Shared<DockerLayer>> layers; + + Fetcher* fetcher; +}; + +} // namespace docker { +} // namespace slave { +} // namespace internal { +} // namespace mesos { + +#endif // __MESOS_DOCKER_STORE__ http://git-wip-us.apache.org/repos/asf/mesos/blob/38319bfb/src/slave/flags.cpp ---------------------------------------------------------------------- diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp index 8792162..5406ef8 100644 --- a/src/slave/flags.cpp +++ b/src/slave/flags.cpp @@ -93,6 +93,11 @@ mesos::internal::slave::Flags::Flags() "Directory the appc provisioner will store images in.", "/tmp/mesos/store/appc"); + add(&Flags::docker_store_dir, + "docker_store_dir", + "Directory the docker provisioner will store images in", + "/tmp/mesos/store/docker"); + add(&Flags::default_role, "default_role", "Any resources in the --resources flag that\n" http://git-wip-us.apache.org/repos/asf/mesos/blob/38319bfb/src/slave/flags.hpp ---------------------------------------------------------------------- diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp index 3f6601a..1bfb447 100644 --- a/src/slave/flags.hpp +++ b/src/slave/flags.hpp @@ -53,6 +53,7 @@ public: std::string image_provisioner_backend; std::string appc_store_dir; + std::string docker_store_dir; std::string default_role; Option<std::string> attributes; Bytes fetcher_cache_size;
