Added Docker image reference store. Review: https://reviews.apache.org/r/37247
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/e8906a13 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/e8906a13 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/e8906a13 Branch: refs/heads/master Commit: e8906a1339acdf55eb4c03061fbac872f991f447 Parents: 86d87aa Author: Lily Chen <[email protected]> Authored: Tue Jul 28 16:13:00 2015 -0700 Committer: Timothy Chen <[email protected]> Committed: Fri Sep 25 09:02:04 2015 -0700 ---------------------------------------------------------------------- src/Makefile.am | 10 + src/messages/docker_provisioner.hpp | 24 ++ src/messages/docker_provisioner.proto | 35 +++ .../provisioners/docker/reference_store.cpp | 218 +++++++++++++++++++ .../provisioners/docker/reference_store.hpp | 137 ++++++++++++ .../containerizer/provisioners/docker/store.cpp | 31 +-- .../containerizer/provisioners/docker/store.hpp | 7 +- 7 files changed, 430 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/Makefile.am ---------------------------------------------------------------------- diff --git a/src/Makefile.am b/src/Makefile.am index 916be39..0b49a3b 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -249,6 +249,11 @@ STATE_PROTOS = messages/state.pb.cc messages/state.pb.h BUILT_SOURCES += $(STATE_PROTOS) CLEANFILES += $(STATE_PROTOS) +DOCKER_PROVISIONER_PROTOS = messages/docker_provisioner.pb.cc messages/docker_provisioner.pb.h + +BUILT_SOURCES += $(DOCKER_PROVISIONER_PROTOS) +CLEANFILES += $(DOCKER_PROVISIONER_PROTOS) + REGISTRY_PROTOS = master/registry.pb.cc master/registry.pb.h BUILT_SOURCES += $(REGISTRY_PROTOS) @@ -435,6 +440,7 @@ nodist_libmesos_no_3rdparty_la_SOURCES = \ $(CXX_PROTOS) \ $(FLAGS_PROTOS) \ $(MESSAGES_PROTOS) \ + $(DOCKER_PROVISIONER_PROTOS) \ $(REGISTRY_PROTOS) # TODO(tillt): Remove authentication/cram_md5/* which will enable us to @@ -479,6 +485,7 @@ libmesos_no_3rdparty_la_SOURCES = \ master/validation.cpp \ master/allocator/allocator.cpp \ master/allocator/sorter/drf/sorter.cpp \ + messages/docker_provisioner.proto \ messages/flags.proto \ messages/messages.cpp \ messages/messages.proto \ @@ -693,6 +700,7 @@ if OS_LINUX libmesos_no_3rdparty_la_SOURCES += slave/containerizer/linux_launcher.cpp libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/backends/bind.cpp libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/docker.cpp + libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/docker/reference_store.cpp libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/docker/store.cpp else EXTRA_DIST += linux/cgroups.cpp @@ -788,6 +796,7 @@ libmesos_no_3rdparty_la_SOURCES += \ master/allocator/mesos/hierarchical.hpp \ master/allocator/sorter/drf/sorter.hpp \ master/allocator/sorter/sorter.hpp \ + messages/docker_provisioner.hpp \ messages/flags.hpp \ messages/messages.hpp \ module/manager.hpp \ @@ -818,6 +827,7 @@ libmesos_no_3rdparty_la_SOURCES += \ slave/containerizer/provisioner/backends/bind.hpp \ slave/containerizer/provisioner/backends/copy.hpp \ slave/containerizer/provisioner/docker.hpp \ + slave/containerizer/provisioner/docker/reference_store.hpp \ slave/containerizer/provisioner/docker/registry_client.hpp \ slave/containerizer/provisioner/docker/store.hpp \ slave/containerizer/provisioner/docker/token_manager.hpp \ http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/messages/docker_provisioner.hpp ---------------------------------------------------------------------- diff --git a/src/messages/docker_provisioner.hpp b/src/messages/docker_provisioner.hpp new file mode 100644 index 0000000..b076fdb --- /dev/null +++ b/src/messages/docker_provisioner.hpp @@ -0,0 +1,24 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __DOCKER_PROVISIONER_HPP__ +#define __DOCKER_PROVISIONER_HPP__ + +#include "messages/docker_provisioner.pb.h" + +#endif // __DOCKER_PROVISIONER_HPP__ http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/messages/docker_provisioner.proto ---------------------------------------------------------------------- diff --git a/src/messages/docker_provisioner.proto b/src/messages/docker_provisioner.proto new file mode 100644 index 0000000..9de6707 --- /dev/null +++ b/src/messages/docker_provisioner.proto @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "mesos/mesos.proto"; + +package mesos.internal; + +/** + * A Docker Image name and the layer ids of the layers that comprise the image. + * The layerIds are ordered, with the root layer id (no parent layer id) first + * and the leaf layer id last. + */ +message DockerProvisionerImages { + message Image { + required string name = 1; + repeated string layer_ids = 2; + } + + repeated Image images = 1; +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/slave/containerizer/provisioners/docker/reference_store.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioners/docker/reference_store.cpp b/src/slave/containerizer/provisioners/docker/reference_store.cpp new file mode 100644 index 0000000..b435ed4 --- /dev/null +++ b/src/slave/containerizer/provisioners/docker/reference_store.cpp @@ -0,0 +1,218 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <vector> + +#include <glog/logging.h> + +#include <stout/foreach.hpp> +#include <stout/hashset.hpp> +#include <stout/os.hpp> +#include <stout/protobuf.hpp> + +#include <process/defer.hpp> +#include <process/dispatch.hpp> +#include <process/owned.hpp> + +#include "common/status_utils.hpp" + +#include "messages/docker_provisioner.hpp" + +#include "slave/containerizer/provisioners/docker/reference_store.hpp" +#include "slave/state.hpp" + +using namespace process; + +using std::list; +using std::string; +using std::vector; + +namespace mesos { +namespace internal { +namespace slave { +namespace docker { + +Try<Owned<ReferenceStore>> ReferenceStore::create(const Flags& flags) +{ + Try<Owned<ReferenceStoreProcess>> process = + ReferenceStoreProcess::create(flags); + if (process.isError()) { + return Error("Failed to create reference store: " + process.error()); + } + return Owned<ReferenceStore>(new ReferenceStore(process.get())); +} + + +ReferenceStore::ReferenceStore(Owned<ReferenceStoreProcess> process) + : process(process) +{ + process::spawn(CHECK_NOTNULL(process.get())); +} + + +ReferenceStore::~ReferenceStore() +{ + process::terminate(process.get()); + process::wait(process.get()); +} + + +void ReferenceStore::initialize() +{ + process::dispatch(process.get(), &ReferenceStoreProcess::initialize); +} + + +Future<DockerImage> ReferenceStore::put( + const string& name, + const list<string>& layers) +{ + return dispatch( + process.get(), &ReferenceStoreProcess::put, name, layers); +} + + +Future<Option<DockerImage>> ReferenceStore::get(const string& name) +{ + return dispatch(process.get(), &ReferenceStoreProcess::get, name); +} + + +ReferenceStoreProcess::ReferenceStoreProcess(const Flags& flags) + : flags(flags) {} + + +Try<Owned<ReferenceStoreProcess>> ReferenceStoreProcess::create( + const Flags& flags) +{ + Owned<ReferenceStoreProcess> referenceStore = + Owned<ReferenceStoreProcess>(new ReferenceStoreProcess(flags)); + + return referenceStore; +} + + +Future<DockerImage> ReferenceStoreProcess::put( + const string& name, + const list<string>& layers) +{ + storedImages[name] = DockerImage(name, layers); + + Try<Nothing> status = persist(); + if (status.isError()) { + return Failure("Failed to save state of Docker images" + status.error()); + } + + return storedImages[name]; +} + + +Future<Option<DockerImage>> ReferenceStoreProcess::get(const string& name) +{ + if (!storedImages.contains(name)) { + return None(); + } + + return storedImages[name]; +} + + +Try<Nothing> ReferenceStoreProcess::persist() +{ + DockerProvisionerImages images; + + foreachpair( + const string& name, const DockerImage& dockerImage, storedImages) { + DockerProvisionerImages::Image* image = images.add_images(); + + image->set_name(name); + + foreach (const string& layer, dockerImage.layers) { + image->add_layer_ids(layer); + } + } + + Try<string> path = path::join(flags.docker_store_dir, "storedImages"); + if (path.isError()) { + return Error("Failure to construct path to repositories lookup: " + + path.error()); + } + + Try<Nothing> status = + mesos::internal::slave::state::checkpoint(path.get(), images); + if (status.isError()) { + return Error("Failed to perform checkpoint: " + status.error()); + } + + return Nothing(); +} + + +void ReferenceStoreProcess::initialize() +{ + Try<string> path = path::join(flags.docker_store_dir, "storedImages"); + + storedImages.clear(); + if (!os::exists(path.get())) { + LOG(INFO) << "No images to load from disk. Docker provisioner image " + << "storage path: " << path.get() << " does not exist."; + return; + } + + Result<DockerProvisionerImages> images = + ::protobuf::read<DockerProvisionerImages>(path.get()); + if (images.isError()) { + LOG(ERROR) << "Failed to read protobuf for Docker provisioner image: " + << images.error(); + return; + } + + for (int i = 0; i < images.get().images_size(); i++) { + string imageName = images.get().images(i).name(); + + list<string> layers; + vector<string> missingLayers; + for (int j = 0; j < images.get().images(i).layer_ids_size(); j++) { + string layerId = images.get().images(i).layer_ids(j); + + layers.push_back(layerId); + + if (!os::exists(path::join(flags.docker_store_dir, layerId))) { + missingLayers.push_back(layerId); + } + } + + if (!missingLayers.empty()) { + foreach (const string& layer, missingLayers) { + LOG(WARNING) << "Image layer: " << layer << " required for Docker " + << "image: " << imageName << " is not on disk."; + } + LOG(WARNING) << "Skipped loading image: " << imageName + << " due to missing layers."; + continue; + } + + VLOG(1) << "Loaded Docker image: " << imageName << " from disk."; + storedImages[imageName] = DockerImage(imageName, layers); + } +} + +} // namespace docker { +} // namespace slave { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/slave/containerizer/provisioners/docker/reference_store.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioners/docker/reference_store.hpp b/src/slave/containerizer/provisioners/docker/reference_store.hpp new file mode 100644 index 0000000..d9f7070 --- /dev/null +++ b/src/slave/containerizer/provisioners/docker/reference_store.hpp @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __MESOS_DOCKER_REFERENCE_STORE__ +#define __MESOS_DOCKER_REFERENCE_STORE__ + +#include <list> +#include <string> + +#include <stout/hashmap.hpp> +#include <stout/json.hpp> +#include <stout/option.hpp> +#include <stout/protobuf.hpp> +#include <stout/try.hpp> + +#include <process/future.hpp> +#include <process/owned.hpp> +#include <process/process.hpp> + +#include "slave/containerizer/provisioners/docker.hpp" +#include "slave/flags.hpp" + +namespace mesos { +namespace internal { +namespace slave { +namespace docker { + +// Forward Declaration. +class ReferenceStoreProcess; + +/** + * The Reference Store is a way to track the Docker images used by the + * provisioner that are stored in on disk. It keeps track of the layers + * that Docker images are composed of and recovers DockerImage objects upon + * initialization by checking for dependent layers stored on disk. + * Currently, image layers are stored indefinitely, with no garbage collection + * of unreferenced image layers. + */ +class ReferenceStore +{ +public: + ~ReferenceStore(); + + /** + * Recover all Docker Images that are on disk by checking if all + * layer dependencies for that layer are present on disk. + */ + void initialize(); + + static Try<process::Owned<ReferenceStore>> create(const Flags& flags); + + /** + * Create a DockerImage, put it in reference store and persist the reference + * store state to disk. + * + * @param name the name of the Docker image to place in the reference store. + * @param layers the list of layer ids that comprise the Docker image in + * order where the root layer's id (no parent layer) is first + * and the leaf layer's id is last. + */ + process::Future<DockerImage> put( + const std::string& name, + const std::list<std::string>& layers); + + /** + * Retrieve DockerImage based on image name if it is among the DockerImages + * stored in memory. + * + * @param name the name of the Docker image to retrieve + */ + process::Future<Option<DockerImage>> get(const std::string& name); + +private: + explicit ReferenceStore(process::Owned<ReferenceStoreProcess> process); + + ReferenceStore(const ReferenceStore&); // Not copyable. + ReferenceStore& operator=(const ReferenceStore&); // Not assignable. + + process::Owned<ReferenceStoreProcess> process; +}; + + +class ReferenceStoreProcess : public process::Process<ReferenceStoreProcess> +{ +public: + ~ReferenceStoreProcess() {} + + // Explicitly use 'initialize' since we are overloading below. + using process::ProcessBase::initialize; + + void initialize(); + + static Try<process::Owned<ReferenceStoreProcess>> create(const Flags& flags); + + process::Future<DockerImage> put( + const std::string& name, + const std::list<std::string>& layers); + + process::Future<Option<DockerImage>> get(const std::string& name); + + // TODO(chenlily): Implement removal of unreferenced images. + +private: + ReferenceStoreProcess(const Flags& flags); + + // Write out reference store state to persistent store. + Try<Nothing> persist(); + + const Flags flags; + + // This is a lookup table for images that are stored in memory. It is keyed + // by the name of the DockerImage. + // For example, "ubuntu:14.04" -> ubuntu14:04 DockerImage. + hashmap<std::string, DockerImage> storedImages; +}; + +} // namespace docker { +} // namespace slave { +} // namespace internal { +} // namespace mesos { + +#endif // __MESOS_DOCKER_REFERENCE_STORE__ http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/slave/containerizer/provisioners/docker/store.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioners/docker/store.cpp b/src/slave/containerizer/provisioners/docker/store.cpp index 9453d6f..b902f8d 100644 --- a/src/slave/containerizer/provisioners/docker/store.cpp +++ b/src/slave/containerizer/provisioners/docker/store.cpp @@ -107,20 +107,12 @@ Try<Owned<LocalStoreProcess>> LocalStoreProcess::create( const Flags& flags, Fetcher* fetcher) { - Owned<LocalStoreProcess> store = - Owned<LocalStoreProcess>(new LocalStoreProcess(flags)); - - Try<Nothing> restore = store->restore(flags); - if (restore.isError()) { - return Error("Failed to restore store: " + restore.error()); - } - - return store; + return Owned<LocalStoreProcess>(new LocalStoreProcess(flags)); } LocalStoreProcess::LocalStoreProcess(const Flags& flags) - : flags(flags) {} + : flags(flags), refStore(ReferenceStore::create(flags).get()) {} Future<DockerImage> LocalStoreProcess::put( @@ -263,10 +255,7 @@ Future<DockerImage> LocalStoreProcess::putImage( return putLayers(staging, layers, sandbox) .then([=]() -> Future<DockerImage> { - images[name] = DockerImage(name, layers); - - // TODO(chenlily): update reference store or replace with reference store - return images[name]; + return refStore->put(name, layers); }); } @@ -415,19 +404,7 @@ Future<Nothing> LocalStoreProcess::moveLayer( Future<Option<DockerImage>> LocalStoreProcess::get(const string& name) { - if (!images.contains(name)) { - return None(); - } - - return images[name]; -} - - -// Recover stored image layers and update layers map. -// TODO(chenlily): Implement restore. -Try<Nothing> LocalStoreProcess::restore(const Flags& flags) -{ - return Nothing(); + return refStore->get(name); } } // namespace docker { http://git-wip-us.apache.org/repos/asf/mesos/blob/e8906a13/src/slave/containerizer/provisioners/docker/store.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioners/docker/store.hpp b/src/slave/containerizer/provisioners/docker/store.hpp index 043f2d7..256e146 100644 --- a/src/slave/containerizer/provisioners/docker/store.hpp +++ b/src/slave/containerizer/provisioners/docker/store.hpp @@ -35,6 +35,7 @@ #include "slave/containerizer/fetcher.hpp" #include "slave/containerizer/provisioners/docker.hpp" +#include "slave/containerizer/provisioners/docker/reference_store.hpp" #include "slave/flags.hpp" namespace mesos { @@ -133,8 +134,6 @@ public: private: LocalStoreProcess(const Flags& flags); - Try<Nothing> restore(const Flags& flags); - process::Future<Nothing> untarImage( const std::string& tarPath, const std::string& staging); @@ -165,9 +164,7 @@ private: const Flags flags; - // This hashmap maps a Docker image by name to its corresponding DockerImage - // object. - hashmap<std::string, DockerImage> images; + process::Owned<ReferenceStore> refStore; }; } // namespace docker {
