Moved files to prepare for unifying provisioners. Review: https://reviews.apache.org/r/38407
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/cc1f8f54 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/cc1f8f54 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/cc1f8f54 Branch: refs/heads/master Commit: cc1f8f54ec1373e500c4a4b25b92a455d2a27a83 Parents: c68c3bd Author: Jie Yu <[email protected]> Authored: Tue Sep 15 11:01:20 2015 -0700 Committer: Jie Yu <[email protected]> Committed: Wed Sep 16 15:45:49 2015 -0700 ---------------------------------------------------------------------- src/Makefile.am | 50 +- .../isolators/filesystem/linux.hpp | 3 +- src/slave/containerizer/mesos/containerizer.cpp | 3 +- src/slave/containerizer/provisioner.cpp | 79 --- src/slave/containerizer/provisioner.hpp | 79 --- .../containerizer/provisioner/appc/paths.cpp | 85 +++ .../containerizer/provisioner/appc/paths.hpp | 83 +++ .../provisioner/appc/provisioner.cpp | 397 +++++++++++ .../provisioner/appc/provisioner.hpp | 78 +++ .../containerizer/provisioner/appc/spec.cpp | 104 +++ .../containerizer/provisioner/appc/spec.hpp | 54 ++ .../containerizer/provisioner/appc/store.cpp | 280 ++++++++ .../containerizer/provisioner/appc/store.hpp | 90 +++ src/slave/containerizer/provisioner/backend.cpp | 62 ++ src/slave/containerizer/provisioner/backend.hpp | 67 ++ .../containerizer/provisioner/backends/bind.cpp | 197 ++++++ .../containerizer/provisioner/backends/bind.hpp | 75 ++ .../containerizer/provisioner/backends/copy.cpp | 203 ++++++ .../containerizer/provisioner/backends/copy.hpp | 61 ++ .../provisioner/docker/registry_client.cpp | 575 ++++++++++++++++ .../provisioner/docker/registry_client.hpp | 163 +++++ .../provisioner/docker/token_manager.cpp | 361 ++++++++++ .../provisioner/docker/token_manager.hpp | 179 +++++ src/slave/containerizer/provisioner/paths.cpp | 192 ++++++ src/slave/containerizer/provisioner/paths.hpp | 83 +++ .../containerizer/provisioner/provisioner.cpp | 79 +++ .../containerizer/provisioner/provisioner.hpp | 79 +++ .../containerizer/provisioners/appc/paths.cpp | 85 --- .../containerizer/provisioners/appc/paths.hpp | 83 --- .../provisioners/appc/provisioner.cpp | 397 ----------- .../provisioners/appc/provisioner.hpp | 78 --- .../containerizer/provisioners/appc/spec.cpp | 104 --- .../containerizer/provisioners/appc/spec.hpp | 54 -- .../containerizer/provisioners/appc/store.cpp | 280 -------- .../containerizer/provisioners/appc/store.hpp | 90 --- .../containerizer/provisioners/backend.cpp | 62 -- .../containerizer/provisioners/backend.hpp | 67 -- .../provisioners/backends/bind.cpp | 197 ------ .../provisioners/backends/bind.hpp | 75 -- .../provisioners/backends/copy.cpp | 203 ------ .../provisioners/backends/copy.hpp | 61 -- .../provisioners/docker/registry_client.cpp | 575 ---------------- .../provisioners/docker/registry_client.hpp | 163 ----- .../provisioners/docker/token_manager.cpp | 361 ---------- .../provisioners/docker/token_manager.hpp | 179 ----- src/slave/containerizer/provisioners/paths.cpp | 191 ----- src/slave/containerizer/provisioners/paths.hpp | 83 --- .../containerizer/appc_provisioner_tests.cpp | 415 ----------- .../containerizer/docker_provisioner_tests.cpp | 688 ------------------- src/tests/containerizer/provisioner.hpp | 2 +- .../containerizer/provisioner_appc_tests.cpp | 415 +++++++++++ .../containerizer/provisioner_backend_tests.cpp | 4 +- .../containerizer/provisioner_docker_tests.cpp | 688 +++++++++++++++++++ 53 files changed, 4682 insertions(+), 4679 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/Makefile.am ---------------------------------------------------------------------- diff --git a/src/Makefile.am b/src/Makefile.am index db2f8d9..60cb10d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -503,16 +503,16 @@ libmesos_no_3rdparty_la_SOURCES = \ slave/containerizer/launcher.cpp \ slave/containerizer/mesos/containerizer.cpp \ slave/containerizer/mesos/launch.cpp \ - slave/containerizer/provisioner.cpp \ - slave/containerizer/provisioners/paths.cpp \ - slave/containerizer/provisioners/appc/paths.cpp \ - slave/containerizer/provisioners/appc/provisioner.cpp \ - slave/containerizer/provisioners/appc/spec.cpp \ - slave/containerizer/provisioners/appc/store.cpp \ - slave/containerizer/provisioners/backend.cpp \ - slave/containerizer/provisioners/backends/copy.cpp \ - slave/containerizer/provisioners/docker/registry_client.cpp \ - slave/containerizer/provisioners/docker/token_manager.cpp \ + slave/containerizer/provisioner/paths.cpp \ + slave/containerizer/provisioner/provisioner.cpp \ + slave/containerizer/provisioner/appc/paths.cpp \ + slave/containerizer/provisioner/appc/provisioner.cpp \ + slave/containerizer/provisioner/appc/spec.cpp \ + slave/containerizer/provisioner/appc/store.cpp \ + slave/containerizer/provisioner/backend.cpp \ + slave/containerizer/provisioner/backends/copy.cpp \ + slave/containerizer/provisioner/docker/registry_client.cpp \ + slave/containerizer/provisioner/docker/token_manager.cpp \ slave/resource_estimators/noop.cpp \ usage/usage.cpp \ v1/attributes.cpp \ @@ -680,7 +680,7 @@ if OS_LINUX libmesos_no_3rdparty_la_SOURCES += slave/containerizer/isolators/filesystem/linux.cpp libmesos_no_3rdparty_la_SOURCES += slave/containerizer/isolators/filesystem/shared.cpp libmesos_no_3rdparty_la_SOURCES += slave/containerizer/linux_launcher.cpp - libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioners/backends/bind.cpp + libmesos_no_3rdparty_la_SOURCES += slave/containerizer/provisioner/backends/bind.cpp else EXTRA_DIST += linux/cgroups.cpp EXTRA_DIST += linux/fs.cpp @@ -794,17 +794,17 @@ libmesos_no_3rdparty_la_SOURCES += \ slave/containerizer/isolator.hpp \ slave/containerizer/launcher.hpp \ slave/containerizer/linux_launcher.hpp \ - slave/containerizer/provisioner.hpp \ - slave/containerizer/provisioners/paths.hpp \ - slave/containerizer/provisioners/appc/paths.hpp \ - slave/containerizer/provisioners/appc/provisioner.hpp \ - slave/containerizer/provisioners/appc/spec.hpp \ - slave/containerizer/provisioners/appc/store.hpp \ - slave/containerizer/provisioners/backend.hpp \ - slave/containerizer/provisioners/backends/bind.hpp \ - slave/containerizer/provisioners/backends/copy.hpp \ - slave/containerizer/provisioners/docker/registry_client.hpp \ - slave/containerizer/provisioners/docker/token_manager.hpp \ + slave/containerizer/provisioner/paths.hpp \ + slave/containerizer/provisioner/provisioner.hpp \ + slave/containerizer/provisioner/appc/paths.hpp \ + slave/containerizer/provisioner/appc/provisioner.hpp \ + slave/containerizer/provisioner/appc/spec.hpp \ + slave/containerizer/provisioner/appc/store.hpp \ + slave/containerizer/provisioner/backend.hpp \ + slave/containerizer/provisioner/backends/bind.hpp \ + slave/containerizer/provisioner/backends/copy.hpp \ + slave/containerizer/provisioner/docker/registry_client.hpp \ + slave/containerizer/provisioner/docker/token_manager.hpp \ slave/containerizer/isolators/posix.hpp \ slave/containerizer/isolators/posix/disk.hpp \ slave/containerizer/isolators/cgroups/constants.hpp \ @@ -1719,16 +1719,16 @@ mesos_tests_SOURCES = \ tests/zookeeper_url_tests.cpp \ tests/common/http_tests.cpp \ tests/common/recordio_tests.cpp \ - tests/containerizer/appc_provisioner_tests.cpp \ tests/containerizer/composing_containerizer_tests.cpp \ tests/containerizer/docker_containerizer_tests.cpp \ tests/containerizer/docker_tests.cpp \ - tests/containerizer/docker_provisioner_tests.cpp \ tests/containerizer/external_containerizer_test.cpp \ tests/containerizer/isolator_tests.cpp \ tests/containerizer/memory_test_helper.cpp \ tests/containerizer/mesos_containerizer_tests.cpp \ - tests/containerizer/provisioner_backend_tests.cpp + tests/containerizer/provisioner_appc_tests.cpp \ + tests/containerizer/provisioner_backend_tests.cpp \ + tests/containerizer/provisioner_docker_tests.cpp mesos_tests_CPPFLAGS = $(MESOS_CPPFLAGS) mesos_tests_CPPFLAGS += -DSOURCE_DIR=\"$(abs_top_srcdir)\" http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/isolators/filesystem/linux.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/isolators/filesystem/linux.hpp b/src/slave/containerizer/isolators/filesystem/linux.hpp index 6cfe9fa..ff76c89 100644 --- a/src/slave/containerizer/isolators/filesystem/linux.hpp +++ b/src/slave/containerizer/isolators/filesystem/linux.hpp @@ -29,7 +29,8 @@ #include "slave/flags.hpp" #include "slave/containerizer/isolator.hpp" -#include "slave/containerizer/provisioner.hpp" + +#include "slave/containerizer/provisioner/provisioner.hpp" namespace mesos { namespace internal { http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/mesos/containerizer.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/containerizer.cpp b/src/slave/containerizer/mesos/containerizer.cpp index 1b83a87..6ab4c08 100644 --- a/src/slave/containerizer/mesos/containerizer.cpp +++ b/src/slave/containerizer/mesos/containerizer.cpp @@ -46,7 +46,6 @@ #ifdef __linux__ #include "slave/containerizer/linux_launcher.hpp" #endif -#include "slave/containerizer/provisioner.hpp" #include "slave/containerizer/isolators/posix.hpp" @@ -77,6 +76,8 @@ #include "slave/containerizer/mesos/containerizer.hpp" #include "slave/containerizer/mesos/launch.hpp" +#include "slave/containerizer/provisioner/provisioner.hpp" + using std::list; using std::map; using std::string; http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner.cpp b/src/slave/containerizer/provisioner.cpp deleted file mode 100644 index 2ac9008..0000000 --- a/src/slave/containerizer/provisioner.cpp +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include <stout/hashset.hpp> -#include <stout/stringify.hpp> -#include <stout/strings.hpp> - -#include "slave/containerizer/provisioner.hpp" - -#include "slave/containerizer/provisioners/appc/provisioner.hpp" - -using namespace process; - -using std::string; - -namespace mesos { -namespace internal { -namespace slave { - -Try<hashmap<Image::Type, Owned<Provisioner>>> Provisioner::create( - const Flags& flags, - Fetcher* fetcher) -{ - if (flags.provisioners.isNone()) { - return hashmap<Image::Type, Owned<Provisioner>>(); - } - - hashmap<Image::Type, - Try<Owned<Provisioner>>(*)(const Flags&, Fetcher*)> creators; - - // Register all supported creators. - creators.put(Image::APPC, &appc::AppcProvisioner::create); - - hashmap<Image::Type, Owned<Provisioner>> provisioners; - - // NOTE: Change in '--provisioners' flag may result in leaked rootfs - // files on the disk but it's at least safe because files managed by - // different provisioners are totally separated. - foreach (const string& type, - strings::tokenize(flags.provisioners.get(), ",")) { - Image::Type imageType; - if (!Image::Type_Parse(strings::upper(type), &imageType)) { - return Error("Unknown provisioner '" + type + "'"); - } - - if (!creators.contains(imageType)) { - return Error("Unsupported provisioner '" + type + "'"); - } - - Try<Owned<Provisioner>> provisioner = creators[imageType](flags, fetcher); - if (provisioner.isError()) { - return Error("Failed to create '" + stringify(imageType) + - "' provisioner: " + provisioner.error()); - } - - provisioners[imageType] = provisioner.get(); - } - - return provisioners; -} - -} // namespace slave { -} // namespace internal { -} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner.hpp b/src/slave/containerizer/provisioner.hpp deleted file mode 100644 index 9e0e0b8..0000000 --- a/src/slave/containerizer/provisioner.hpp +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __MESOS_PROVISIONER_HPP__ -#define __MESOS_PROVISIONER_HPP__ - -#include <list> - -#include <mesos/resources.hpp> - -#include <mesos/slave/isolator.hpp> // For ContainerState. - -#include <stout/hashmap.hpp> -#include <stout/nothing.hpp> -#include <stout/try.hpp> - -#include <process/future.hpp> -#include <process/owned.hpp> - -#include "slave/flags.hpp" - -#include "slave/containerizer/fetcher.hpp" - -namespace mesos { -namespace internal { -namespace slave { - -class Provisioner -{ -public: - virtual ~Provisioner() {} - - // Create provisioners based on specified flags. An error is returned if - // any of the provisioners specified in --provisioner failed to be created. - static Try<hashmap<Image::Type, process::Owned<Provisioner>>> - create(const Flags& flags, Fetcher* fetcher); - - // Recover root filesystems for containers from the run states and - // the orphan containers (known to the launcher but not known to the - // slave) detected by the launcher. This function is also - // responsible for cleaning up any intermediate artifacts (e.g. - // directories) to not leak anything. - virtual process::Future<Nothing> recover( - const std::list<mesos::slave::ContainerState>& states, - const hashset<ContainerID>& orphans) = 0; - - // Provision a root filesystem for the container using the specified - // image and return the absolute path to the root filesystem. - virtual process::Future<std::string> provision( - const ContainerID& containerId, - const Image& image) = 0; - - // Destroy a previously provisioned root filesystem. Assumes that - // all references (e.g., mounts, open files) to the provisioned - // filesystem have been removed. Return false if there is no - // provisioned root filesystem for the given container. - virtual process::Future<bool> destroy(const ContainerID& containerId) = 0; -}; - -} // namespace slave { -} // namespace internal { -} // namespace mesos { - -#endif // __MESOS_PROVISIONER_HPP__ http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/appc/paths.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/appc/paths.cpp b/src/slave/containerizer/provisioner/appc/paths.cpp new file mode 100644 index 0000000..8817c0f --- /dev/null +++ b/src/slave/containerizer/provisioner/appc/paths.cpp @@ -0,0 +1,85 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <list> + +#include <glog/logging.h> + +#include <stout/path.hpp> + +#include "slave/containerizer/provisioner/appc/paths.hpp" + +using std::list; +using std::string; + +namespace mesos { +namespace internal { +namespace slave { +namespace appc { +namespace paths { + +string getStagingDir(const string& storeDir) +{ + return path::join(storeDir, "staging"); +} + + +string getImagesDir(const string& storeDir) +{ + return path::join(storeDir, "images"); +} + + +string getImagePath(const string& storeDir, const string& imageId) +{ + return path::join(getImagesDir(storeDir), imageId); +} + + +string getImageRootfsPath( + const string& storeDir, + const string& imageId) +{ + return path::join(getImagePath(storeDir, imageId), "rootfs"); +} + + +string getImageRootfsPath(const string& imagePath) +{ + return path::join(imagePath, "rootfs"); +} + + +string getImageManifestPath( + const string& storeDir, + const string& imageId) +{ + return path::join(getImagePath(storeDir, imageId), "manifest"); +} + + +string getImageManifestPath(const string& imagePath) +{ + return path::join(imagePath, "manifest"); +} + +} // namespace paths { +} // namespace appc { +} // namespace slave { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/appc/paths.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/appc/paths.hpp b/src/slave/containerizer/provisioner/appc/paths.hpp new file mode 100644 index 0000000..7c36d67 --- /dev/null +++ b/src/slave/containerizer/provisioner/appc/paths.hpp @@ -0,0 +1,83 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __PROVISIONER_APPC_PATHS_HPP__ +#define __PROVISIONER_APPC_PATHS_HPP__ + +#include <string> + +#include <mesos/mesos.hpp> + +#include <stout/hashmap.hpp> +#include <stout/try.hpp> + +namespace mesos { +namespace internal { +namespace slave { +namespace appc { +namespace paths { + +// The appc store file system layout is as follows: +// +// <store_dir> ('--appc_store_dir' flag) +// |--staging (contains temp directories for staging downloads) +// | +// |--images (stores validated images) +// |--<image_id> (in the form of "sha512-<128_character_hash_sum>") +// |--manifest +// |--rootfs +// |--... (according to the ACI spec) +// +// TODO(xujyan): The staging directory is unused for now (it's +// externally managed) but implemented to illustrate the need for a +// separate 'images' directory. Complete the layout diagram when the +// staging directory is utilized by the provisioner. + +std::string getStagingDir(const std::string& storeDir); + + +std::string getImagesDir(const std::string& storeDir); + + +std::string getImagePath( + const std::string& storeDir, + const std::string& imageId); + + +std::string getImageRootfsPath( + const std::string& storeDir, + const std::string& imageId); + + +std::string getImageRootfsPath(const std::string& imagePath); + + +std::string getImageManifestPath( + const std::string& storeDir, + const std::string& imageId); + + +std::string getImageManifestPath(const std::string& imagePath); + +} // namespace paths { +} // namespace appc { +} // namespace slave { +} // namespace internal { +} // namespace mesos { + +#endif // __PROVISIONER_APPC_PATHS_HPP__ http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/appc/provisioner.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/appc/provisioner.cpp b/src/slave/containerizer/provisioner/appc/provisioner.cpp new file mode 100644 index 0000000..929e42a --- /dev/null +++ b/src/slave/containerizer/provisioner/appc/provisioner.cpp @@ -0,0 +1,397 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <mesos/type_utils.hpp> + +#include <process/collect.hpp> +#include <process/defer.hpp> +#include <process/dispatch.hpp> +#include <process/process.hpp> + +#include <stout/foreach.hpp> +#include <stout/hashset.hpp> +#include <stout/os.hpp> +#include <stout/stringify.hpp> +#include <stout/strings.hpp> +#include <stout/uuid.hpp> + +#include "slave/containerizer/provisioner/backend.hpp" +#include "slave/containerizer/provisioner/paths.hpp" + +#include "slave/containerizer/provisioner/appc/paths.hpp" +#include "slave/containerizer/provisioner/appc/provisioner.hpp" +#include "slave/containerizer/provisioner/appc/spec.hpp" +#include "slave/containerizer/provisioner/appc/store.hpp" + +#include "slave/paths.hpp" + +using namespace process; +using namespace mesos::internal::slave; + +using std::list; +using std::string; +using std::vector; + +using mesos::slave::ContainerState; + +namespace mesos { +namespace internal { +namespace slave { +namespace appc { + +class AppcProvisionerProcess : public Process<AppcProvisionerProcess> +{ +public: + AppcProvisionerProcess( + const Flags& flags, + const string& root, + const Owned<Store>& store, + const hashmap<string, Owned<Backend>>& backends); + + Future<Nothing> recover( + const list<ContainerState>& states, + const hashset<ContainerID>& orphans); + + Future<string> provision(const ContainerID& containerId, const Image& image); + + Future<bool> destroy(const ContainerID& containerId); + +private: + Future<string> _provision(const vector<string>& layers, const string& rootfs); + + const Flags flags; + + // Absolute path to the Appc provisioner root directory. It can be derived + // from '--work_dir' but we keep a separate copy here because we converted + // it into an absolute path so managed rootfs paths match the ones in + // 'mountinfo' (important if mount-based backends are used). + const string root; + + const Owned<Store> store; + const hashmap<string, Owned<Backend>> backends; + + struct Info + { + // Mappings: backend -> rootfsId -> rootfsPath. + hashmap<string, hashmap<string, string>> rootfses; + }; + + hashmap<ContainerID, Owned<Info>> infos; +}; + + +// NOTE: Successful creation of the provisioner means its managed +// directory under --work_dir is also created. +Try<Owned<Provisioner>> AppcProvisioner::create( + const Flags& flags, + Fetcher* fetcher) +{ + string _root = + slave::paths::getProvisionerDir(flags.work_dir, Image::APPC); + + Try<Nothing> mkdir = os::mkdir(_root); + if (mkdir.isError()) { + return Error("Failed to create provisioner root directory '" + + _root + "': " + mkdir.error()); + } + + Result<string> root = os::realpath(_root); + if (root.isError()) { + return Error( + "Failed to resolve the realpath of provisioner root directory '" + + _root + "': " + root.error()); + } + + CHECK_SOME(root); // Can't be None since we just created it. + + Try<Owned<Store>> store = Store::create(flags); + if (store.isError()) { + return Error("Failed to create image store: " + store.error()); + } + + hashmap<string, Owned<Backend>> backends = Backend::create(flags); + if (backends.empty()) { + return Error("No usable provisioner backend created"); + } + + if (!backends.contains(flags.appc_provisioner_backend)) { + return Error("The specified provisioner backend '" + + flags.appc_provisioner_backend + "'is unsupported"); + } + + return Owned<Provisioner>(new AppcProvisioner( + Owned<AppcProvisionerProcess>(new AppcProvisionerProcess( + flags, + root.get(), + store.get(), + backends)))); +} + + +AppcProvisioner::AppcProvisioner(Owned<AppcProvisionerProcess> _process) + : process(_process) +{ + spawn(CHECK_NOTNULL(process.get())); +} + + +AppcProvisioner::~AppcProvisioner() +{ + terminate(process.get()); + wait(process.get()); +} + + +Future<Nothing> AppcProvisioner::recover( + const list<ContainerState>& states, + const hashset<ContainerID>& orphans) +{ + return dispatch( + process.get(), + &AppcProvisionerProcess::recover, + states, + orphans); +} + + +Future<string> AppcProvisioner::provision( + const ContainerID& containerId, + const Image& image) +{ + return dispatch( + process.get(), + &AppcProvisionerProcess::provision, + containerId, + image); +} + + +Future<bool> AppcProvisioner::destroy(const ContainerID& containerId) +{ + return dispatch( + process.get(), + &AppcProvisionerProcess::destroy, + containerId); +} + + +AppcProvisionerProcess::AppcProvisionerProcess( + const Flags& _flags, + const string& _root, + const Owned<Store>& _store, + const hashmap<string, Owned<Backend>>& _backends) + : flags(_flags), + root(_root), + store(_store), + backends(_backends) {} + + +Future<Nothing> AppcProvisionerProcess::recover( + const list<ContainerState>& states, + const hashset<ContainerID>& orphans) +{ + // Register living containers, including the ones that do not + // provision Appc images. + hashset<ContainerID> alive; + foreach (const ContainerState& state, states) { + alive.insert(state.container_id()); + } + + // List provisioned containers; recover living ones; destroy unknown orphans. + // Note that known orphan containers are recovered as well and they will + // be destroyed by the containerizer using the normal cleanup path. See + // MESOS-2367 for details. + Try<hashmap<ContainerID, string>> containers = + provisioners::paths::listContainers(root); + + if (containers.isError()) { + return Failure("Failed to list the containers managed by Appc " + "provisioner: " + containers.error()); + } + + // Scan the list of containers, register all of them with 'infos' but + // mark unknown orphans for immediate cleanup. + hashset<ContainerID> unknownOrphans; + foreachkey (const ContainerID& containerId, containers.get()) { + Owned<Info> info = Owned<Info>(new Info()); + + Try<hashmap<string, hashmap<string, string>>> rootfses = + provisioners::paths::listContainerRootfses(root, containerId); + + if (rootfses.isError()) { + return Failure("Unable to list rootfses belonged to container '" + + containerId.value() + "': " + rootfses.error()); + } + + foreachkey (const string& backend, rootfses.get()) { + if (!backends.contains(backend)) { + return Failure("Found rootfses managed by an unrecognized backend: " + + backend); + } + + info->rootfses.put(backend, rootfses.get()[backend]); + } + + infos.put(containerId, info); + + if (alive.contains(containerId) || orphans.contains(containerId)) { + VLOG(1) << "Recovered container " << containerId; + continue; + } else { + // For immediate cleanup below. + unknownOrphans.insert(containerId); + } + } + + LOG(INFO) + << "Recovered living and known orphan containers for Appc provisioner"; + + // Destroy unknown orphan containers' rootfses. + list<Future<bool>> destroys; + foreach (const ContainerID& containerId, unknownOrphans) { + destroys.push_back(destroy(containerId)); + } + + Future<Nothing> cleanup = collect(destroys) + .then([]() -> Future<Nothing> { + LOG(INFO) << "Cleaned up unknown orphan containers for Appc provisioner"; + return Nothing(); + }); + + Future<Nothing> recover = store->recover() + .then([]() -> Future<Nothing> { + LOG(INFO) << "Recovered Appc image store"; + return Nothing(); + }); + + + // A successful provisioner recovery depends on: + // 1) Recovery of living containers and known orphans (done above). + // 2) Successful cleanup of unknown orphans. + // 3) Successful store recovery. + return collect(cleanup, recover) + .then([=]() -> Future<Nothing> { + return Nothing(); + }); +} + + +Future<string> AppcProvisionerProcess::provision( + const ContainerID& containerId, + const Image& image) +{ + if (image.type() != Image::APPC) { + return Failure("Unsupported container image type: " + + stringify(image.type())); + } + + if (!image.has_appc()) { + return Failure("Missing Appc image info"); + } + + string rootfsId = UUID::random().toString(); + string rootfs = provisioners::paths::getContainerRootfsDir( + root, containerId, flags.appc_provisioner_backend, rootfsId); + + if (!infos.contains(containerId)) { + infos.put(containerId, Owned<Info>(new Info())); + } + + infos[containerId]->rootfses[flags.appc_provisioner_backend].put( + rootfsId, rootfs); + + // Get and then provision image layers from the store. + return store->get(image.appc()) + .then(defer(self(), &Self::_provision, lambda::_1, rootfs)); +} + + +Future<string> AppcProvisionerProcess::_provision( + const vector<string>& layers, + const string& rootfs) +{ + LOG(INFO) << "Provisioning image layers to rootfs '" << rootfs << "'"; + + CHECK(backends.contains(flags.appc_provisioner_backend)); + return backends.get(flags.appc_provisioner_backend).get()->provision( + layers, + rootfs) + .then([rootfs]() -> Future<string> { return rootfs; }); +} + + +Future<bool> AppcProvisionerProcess::destroy(const ContainerID& containerId) +{ + if (!infos.contains(containerId)) { + LOG(INFO) << "Ignoring destroy request for unknown container: " + << containerId; + + return false; + } + + // Unregister the container first. If destroy() fails, we can rely + // on recover() to retry it later. + Owned<Info> info = infos[containerId]; + infos.erase(containerId); + + list<Future<bool>> futures; + foreachkey (const string& backend, info->rootfses) { + foreachvalue (const string& rootfs, info->rootfses[backend]) { + if (!backends.contains(backend)) { + return Failure("Cannot destroy rootfs '" + rootfs + + "' provisioned by an unknown backend '" + backend + "'"); + } + + LOG(INFO) << "Destroying container rootfs for container '" + << containerId << "' at '" << rootfs << "'"; + + futures.push_back( + backends.get(backend).get()->destroy(rootfs)); + } + } + + // NOTE: We calculate 'containerDir' here so that the following + // lambda does not need to bind 'this'. + string containerDir = + provisioners::paths::getContainerDir(root, containerId); + + // TODO(xujyan): Revisit the usefulness of this return value. + return collect(futures) + .then([containerDir]() -> Future<bool> { + // This should be fairly cheap as the directory should only + // contain a few empty sub-directories at this point. + // + // TODO(jieyu): Currently, it's possible that some directories + // cannot be removed due to EBUSY. EBUSY is caused by the race + // between cleaning up this container and new containers copying + // the host mount table. It's OK to ignore them. The cleanup + // will be retried during slave recovery. + Try<Nothing> rmdir = os::rmdir(containerDir); + if (rmdir.isError()) { + LOG(ERROR) << "Failed to remove the provisioned container directory " + << "at '" << containerDir << "'"; + } + + return true; + }); +} + +} // namespace appc { +} // namespace slave { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/appc/provisioner.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/appc/provisioner.hpp b/src/slave/containerizer/provisioner/appc/provisioner.hpp new file mode 100644 index 0000000..e4d5b8e --- /dev/null +++ b/src/slave/containerizer/provisioner/appc/provisioner.hpp @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __APPC_PROVISIONER_HPP__ +#define __APPC_PROVISIONER_HPP__ + +#include <list> +#include <string> +#include <vector> + +#include <process/future.hpp> +#include <process/owned.hpp> + +#include <stout/hashmap.hpp> +#include <stout/json.hpp> +#include <stout/nothing.hpp> +#include <stout/try.hpp> + +#include "slave/containerizer/provisioner/provisioner.hpp" + +namespace mesos { +namespace internal { +namespace slave { +namespace appc { + +// Forward declaration. +class AppcProvisionerProcess; + + +class AppcProvisioner : public Provisioner +{ +public: + static Try<process::Owned<Provisioner>> create( + const Flags& flags, + Fetcher* fetcher); + + ~AppcProvisioner(); + + virtual process::Future<Nothing> recover( + const std::list<mesos::slave::ContainerState>& states, + const hashset<ContainerID>& orphans); + + virtual process::Future<std::string> provision( + const ContainerID& containerId, + const Image& image); + + virtual process::Future<bool> destroy(const ContainerID& containerId); + +private: + explicit AppcProvisioner(process::Owned<AppcProvisionerProcess> process); + + AppcProvisioner(const AppcProvisioner&); // Not copyable. + AppcProvisioner& operator=(const AppcProvisioner&); // Not assignable. + + process::Owned<AppcProvisionerProcess> process; +}; + +} // namespace appc { +} // namespace slave { +} // namespace internal { +} // namespace mesos { + +#endif // __APPC_PROVISIONER_HPP__ http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/appc/spec.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/appc/spec.cpp b/src/slave/containerizer/provisioner/appc/spec.cpp new file mode 100644 index 0000000..bbe523d --- /dev/null +++ b/src/slave/containerizer/provisioner/appc/spec.cpp @@ -0,0 +1,104 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <stout/os/stat.hpp> +#include <stout/protobuf.hpp> +#include <stout/strings.hpp> + +#include "slave/containerizer/provisioner/appc/paths.hpp" +#include "slave/containerizer/provisioner/appc/spec.hpp" + +using std::string; + +namespace mesos { +namespace internal { +namespace slave { +namespace appc { +namespace spec { + +Option<Error> validateManifest(const AppcImageManifest& manifest) +{ + // TODO(idownes): Validate that required fields are present when + // this cannot be expressed in the protobuf specification, e.g., + // repeated fields with >= 1. + // TODO(xujyan): More thorough type validation: + // https://github.com/appc/spec/blob/master/spec/types.md + if (manifest.ackind() != "ImageManifest") { + return Error("Incorrect acKind field: " + manifest.ackind()); + } + + return None(); +} + + +Option<Error> validateImageID(const string& imageId) +{ + if (!strings::startsWith(imageId, "sha512-")) { + return Error("Image ID needs to start with sha512-"); + } + + string hash = strings::remove(imageId, "sha512-", strings::PREFIX); + if (hash.length() != 128) { + return Error("Invalid hash length for: " + hash); + } + + return None(); +} + + +Option<Error> validateLayout(const string& imagePath) +{ + if (!os::stat::isdir(paths::getImageRootfsPath(imagePath))) { + return Error("No rootfs directory found in image layout"); + } + + if (!os::stat::isfile(paths::getImageManifestPath(imagePath))) { + return Error("No manifest found in image layout"); + } + + return None(); +} + + +Try<AppcImageManifest> parse(const string& value) +{ + Try<JSON::Object> json = JSON::parse<JSON::Object>(value); + if (json.isError()) { + return Error("JSON parse failed: " + json.error()); + } + + Try<AppcImageManifest> manifest = + protobuf::parse<AppcImageManifest>(json.get()); + + if (manifest.isError()) { + return Error("Protobuf parse failed: " + manifest.error()); + } + + Option<Error> error = validateManifest(manifest.get()); + if (error.isSome()) { + return Error("Schema validation failed: " + error.get().message); + } + + return manifest.get(); +} + +} // namespace spec { +} // namespace appc { +} // namespace slave { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/appc/spec.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/appc/spec.hpp b/src/slave/containerizer/provisioner/appc/spec.hpp new file mode 100644 index 0000000..2bc8c6f --- /dev/null +++ b/src/slave/containerizer/provisioner/appc/spec.hpp @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __PROVISIONER_APPC_SPEC_HPP__ +#define __PROVISIONER_APPC_SPEC_HPP__ + +#include <string> + +#include <stout/error.hpp> +#include <stout/option.hpp> + +#include <mesos/mesos.hpp> + +namespace mesos { +namespace internal { +namespace slave { +namespace appc { +namespace spec { + +// Validate if the specified image manifest conforms to the Appc spec. +Option<Error> validateManifest(const AppcImageManifest& manifest); + +// Validate if the specified image ID conforms to the Appc spec. +Option<Error> validateImageID(const std::string& imageId); + +// Validate if the specified image has the disk layout that conforms +// to the Appc spec. +Option<Error> validateLayout(const std::string& imagePath); + +// Parse the AppcImageManifest in the specified JSON string. +Try<AppcImageManifest> parse(const std::string& value); + +} // namespace spec { +} // namespace appc { +} // namespace slave { +} // namespace internal { +} // namespace mesos { + +#endif // __PROVISIONER_APPC_SPEC_HPP__ http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/appc/store.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/appc/store.cpp b/src/slave/containerizer/provisioner/appc/store.cpp new file mode 100644 index 0000000..327ac9b --- /dev/null +++ b/src/slave/containerizer/provisioner/appc/store.cpp @@ -0,0 +1,280 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <list> + +#include <glog/logging.h> + +#include <process/defer.hpp> +#include <process/dispatch.hpp> + +#include <stout/check.hpp> +#include <stout/hashmap.hpp> +#include <stout/os.hpp> +#include <stout/path.hpp> + +#include "slave/containerizer/provisioner/appc/paths.hpp" +#include "slave/containerizer/provisioner/appc/spec.hpp" +#include "slave/containerizer/provisioner/appc/store.hpp" + +using namespace process; + +using std::list; +using std::string; +using std::vector; + +namespace mesos { +namespace internal { +namespace slave { +namespace appc { + +// Defines a locally cached image (which has passed validation). +struct CachedImage +{ + CachedImage( + const AppcImageManifest& _manifest, + const string& _id, + const string& _path) + : manifest(_manifest), id(_id), path(_path) {} + + string rootfs() const + { + return path::join(path, "rootfs"); + } + + const AppcImageManifest manifest; + + // Image ID of the format "sha512-value" where "value" is the hex + // encoded string of the sha512 digest of the uncompressed tar file + // of the image. + const string id; + + // Absolute path to the extracted image. + const string path; +}; + + +// Helper that implements this: +// https://github.com/appc/spec/blob/master/spec/aci.md#dependency-matching +static bool matches(Image::Appc requirements, const CachedImage& candidate) +{ + // The name must match. + if (candidate.manifest.name() != requirements.name()) { + return false; + } + + // If an id is specified the candidate must match. + if (requirements.has_id() && (candidate.id != requirements.id())) { + return false; + } + + // Extract labels for easier comparison, this also weeds out duplicates. + // TODO(xujyan): Detect duplicate labels in image manifest validation + // and Image::Appc validation. + hashmap<string, string> requiredLabels; + foreach (const Label& label, requirements.labels().labels()) { + requiredLabels[label.key()] = label.value(); + } + + hashmap<string, string> candidateLabels; + foreach (const AppcImageManifest::Label& label, + candidate.manifest.labels()) { + candidateLabels[label.name()] = label.value(); + } + + // Any label specified must be present and match in the candidate. + foreachpair (const string& name, + const string& value, + requiredLabels) { + if (!candidateLabels.contains(name) || + candidateLabels.get(name).get() != value) { + return false; + } + } + + return true; +} + + +class StoreProcess : public Process<StoreProcess> +{ +public: + StoreProcess(const string& root); + + ~StoreProcess() {} + + Future<Nothing> recover(); + + Future<vector<string>> get(const Image::Appc& image); + +private: + // Absolute path to the root directory of the store as defined by + // --appc_store_dir. + const string root; + + // Mappings: name -> id -> image. + hashmap<string, hashmap<string, CachedImage>> images; +}; + + +Try<Owned<Store>> Store::create(const Flags& flags) +{ + Try<Nothing> mkdir = os::mkdir(paths::getImagesDir(flags.appc_store_dir)); + if (mkdir.isError()) { + return Error("Failed to create the images directory: " + mkdir.error()); + } + + // Make sure the root path is canonical so all image paths derived + // from it are canonical too. + Result<string> root = os::realpath(flags.appc_store_dir); + if (!root.isSome()) { + // The above mkdir call recursively creates the store directory + // if necessary so it cannot be None here. + CHECK_ERROR(root); + return Error( + "Failed to get the realpath of the store directory: " + root.error()); + } + + return Owned<Store>(new Store( + Owned<StoreProcess>(new StoreProcess(root.get())))); +} + + +Store::Store(Owned<StoreProcess> _process) + : process(_process) +{ + spawn(CHECK_NOTNULL(process.get())); +} + + +Store::~Store() +{ + terminate(process.get()); + wait(process.get()); +} + + +Future<Nothing> Store::recover() +{ + return dispatch(process.get(), &StoreProcess::recover); +} + + +Future<vector<string>> Store::get(const Image::Appc& image) +{ + return dispatch(process.get(), &StoreProcess::get, image); +} + + +StoreProcess::StoreProcess(const string& _root) : root(_root) {} + + +// Implemented as a helper function because it's going to be used for a +// newly downloaded image too. +static Try<CachedImage> createImage(const string& imagePath) +{ + Option<Error> error = spec::validateLayout(imagePath); + if (error.isSome()) { + return Error("Invalid image layout: " + error.get().message); + } + + string imageId = Path(imagePath).basename(); + + error = spec::validateImageID(imageId); + if (error.isSome()) { + return Error("Invalid image ID: " + error.get().message); + } + + Try<string> read = os::read(paths::getImageManifestPath(imagePath)); + if (read.isError()) { + return Error("Failed to read manifest: " + read.error()); + } + + Try<AppcImageManifest> manifest = spec::parse(read.get()); + if (manifest.isError()) { + return Error("Failed to parse manifest: " + manifest.error()); + } + + return CachedImage(manifest.get(), imageId, imagePath); +} + + +Future<vector<string>> StoreProcess::get(const Image::Appc& image) +{ + if (!images.contains(image.name())) { + return Failure("No image named '" + image.name() + "' can be found"); + } + + // Get local candidates. + vector<CachedImage> candidates; + foreach (const CachedImage& candidate, images[image.name()].values()) { + // The first match is returned. + // TODO(xujyan): Some tie-breaking rules are necessary. + if (matches(image, candidate)) { + LOG(INFO) << "Found match for image '" << image.name() + << "' in the store"; + + // The Appc store current doesn't support dependencies and this is + // enforced by manifest validation: if the image's manifest contains + // dependencies it would fail the validation and wouldn't be stored + // in the store. + return vector<string>({candidate.rootfs()}); + } + } + + return Failure("No image named '" + image.name() + + "' can match the requirements"); +} + + +Future<Nothing> StoreProcess::recover() +{ + // Recover everything in the store. + Try<list<string>> imageIds = os::ls(paths::getImagesDir(root)); + if (imageIds.isError()) { + return Failure( + "Failed to list images under '" + + paths::getImagesDir(root) + "': " + + imageIds.error()); + } + + foreach (const string& imageId, imageIds.get()) { + string path = paths::getImagePath(root, imageId); + if (!os::stat::isdir(path)) { + LOG(WARNING) << "Unexpected entry in storage: " << imageId; + continue; + } + + Try<CachedImage> image = createImage(path); + if (image.isError()) { + LOG(WARNING) << "Unexpected entry in storage: " << image.error(); + continue; + } + + LOG(INFO) << "Restored image '" << image.get().manifest.name() << "'"; + + images[image.get().manifest.name()].put(image.get().id, image.get()); + } + + return Nothing(); +} + +} // namespace appc { +} // namespace slave { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/appc/store.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/appc/store.hpp b/src/slave/containerizer/provisioner/appc/store.hpp new file mode 100644 index 0000000..07218d1 --- /dev/null +++ b/src/slave/containerizer/provisioner/appc/store.hpp @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __PROVISIONER_APPC_STORE_HPP__ +#define __PROVISIONER_APPC_STORE_HPP__ + +#include <string> +#include <vector> + +#include <mesos/mesos.hpp> + +#include <process/future.hpp> +#include <process/owned.hpp> + +#include <stout/try.hpp> + +#include "slave/flags.hpp" + +namespace mesos { +namespace internal { +namespace slave { +namespace appc { + +// Forward declaration. +class StoreProcess; + + +// An image store abstraction that "stores" images. It serves as a read-through +// cache (cache misses are fetched remotely and transparently) for images. +// TODO(xujyan): The store currently keeps cached images indefinitely and we +// should introduce cache eviction policies. +class Store +{ +public: + static Try<process::Owned<Store>> create(const Flags& flags); + + ~Store(); + + process::Future<Nothing> recover(); + + // Get the specified image (and all its recursive dependencies) as a list + // of rootfs layers in the topological order (dependencies go before + // dependents in the list). The images required to build this list are + // either retrieved from the local cache or fetched remotely. + // NOTE: The returned list should not have duplicates. e.g., in the + // following scenario the result should be [C, B, D, A] (B before D in this + // example is decided by the order in which A specifies its dependencies). + // + // A --> B --> C + // | ^ + // |---> D ----| + // + // The returned future fails if the requested image or any of its + // dependencies cannot be found or failed to be fetched. + // TODO(xujyan): Fetching remotely is not implemented for now and until + // then the future fails directly if the image is not in the local cache. + // TODO(xujyan): The store currently doesn't support images that have + // dependencies and we should add it later. + process::Future<std::vector<std::string>> get(const Image::Appc& image); + +private: + Store(process::Owned<StoreProcess> process); + + Store(const Store&); // Not copyable. + Store& operator=(const Store&); // Not assignable. + + process::Owned<StoreProcess> process; +}; + +} // namespace appc { +} // namespace slave { +} // namespace internal { +} // namespace mesos { + +#endif // __PROVISIONER_APPC_STORE_HPP__ http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/backend.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/backend.cpp b/src/slave/containerizer/provisioner/backend.cpp new file mode 100644 index 0000000..b5d9670 --- /dev/null +++ b/src/slave/containerizer/provisioner/backend.cpp @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <glog/logging.h> + +#include <stout/os.hpp> + +#include "slave/containerizer/provisioner/backend.hpp" + +#include "slave/containerizer/provisioner/backends/bind.hpp" +#include "slave/containerizer/provisioner/backends/copy.hpp" + +using namespace process; + +using std::string; + +namespace mesos { +namespace internal { +namespace slave { + +hashmap<string, Owned<Backend>> Backend::create(const Flags& flags) +{ + hashmap<string, Try<Owned<Backend>>(*)(const Flags&)> creators; + +#ifdef __linux__ + creators.put("bind", &BindBackend::create); +#endif // __linux__ + creators.put("copy", &CopyBackend::create); + + hashmap<string, Owned<Backend>> backends; + + foreachkey (const string& name, creators) { + Try<Owned<Backend>> backend = creators[name](flags); + if (backend.isError()) { + LOG(WARNING) << "Failed to create '" << name << "' backend: " + << backend.error(); + continue; + } + backends.put(name, backend.get()); + } + + return backends; +} + +} // namespace slave { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/backend.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/backend.hpp b/src/slave/containerizer/provisioner/backend.hpp new file mode 100644 index 0000000..1c80b79 --- /dev/null +++ b/src/slave/containerizer/provisioner/backend.hpp @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __PROVISIONER_BACKEND_HPP__ +#define __PROVISIONER_BACKEND_HPP__ + +#include <string> +#include <vector> + +#include <process/future.hpp> +#include <process/owned.hpp> + +#include <stout/hashmap.hpp> +#include <stout/try.hpp> + +#include "slave/flags.hpp" + +namespace mesos { +namespace internal { +namespace slave { + +// Provision a root filesystem for a container. +class Backend +{ +public: + virtual ~Backend() {} + + // Return a map of all supported backends keyed by their names. Note + // that Backends that failed to be created due to incorrect flags are + // simply not added to the result. + static hashmap<std::string, process::Owned<Backend>> create( + const Flags& flags); + + // Provision a root filesystem for a container into the specified 'rootfs' + // directory by applying the specified list of root filesystem layers in + // the list order, i.e., files in a layer can overwrite/shadow those from + // another layer earlier in the list. + virtual process::Future<Nothing> provision( + const std::vector<std::string>& layers, + const std::string& rootfs) = 0; + + // Destroy the root filesystem provisioned at the specified 'rootfs' + // directory. Return false if there is no provisioned root filesystem + // to destroy for the given directory. + virtual process::Future<bool> destroy(const std::string& rootfs) = 0; +}; + +} // namespace slave { +} // namespace internal { +} // namespace mesos { + +#endif // __PROVISIONER_BACKEND_HPP__ http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/backends/bind.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/backends/bind.cpp b/src/slave/containerizer/provisioner/backends/bind.cpp new file mode 100644 index 0000000..d853b49 --- /dev/null +++ b/src/slave/containerizer/provisioner/backends/bind.cpp @@ -0,0 +1,197 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <errno.h> +#include <stdio.h> +#include <unistd.h> + +#include <process/dispatch.hpp> +#include <process/process.hpp> + +#include <stout/foreach.hpp> +#include <stout/os.hpp> + +#include "linux/fs.hpp" + +#include "slave/containerizer/provisioner/backends/bind.hpp" + +using namespace process; + +using std::string; +using std::vector; + +namespace mesos { +namespace internal { +namespace slave { + +class BindBackendProcess : public Process<BindBackendProcess> +{ +public: + Future<Nothing> provision(const vector<string>& layers, const string& rootfs); + + Future<bool> destroy(const string& rootfs); +}; + + +Try<Owned<Backend>> BindBackend::create(const Flags&) +{ + Result<string> user = os::user(); + if (!user.isSome()) { + return Error("Failed to determine user: " + + (user.isError() ? user.error() : "username not found")); + } + + if (user.get() != "root") { + return Error("BindBackend requires root privileges"); + } + + return Owned<Backend>(new BindBackend( + Owned<BindBackendProcess>(new BindBackendProcess()))); +} + + +BindBackend::~BindBackend() +{ + terminate(process.get()); + wait(process.get()); +} + + +BindBackend::BindBackend(Owned<BindBackendProcess> _process) + : process(_process) +{ + spawn(CHECK_NOTNULL(process.get())); +} + + +Future<Nothing> BindBackend::provision( + const vector<string>& layers, + const string& rootfs) +{ + return dispatch( + process.get(), &BindBackendProcess::provision, layers, rootfs); +} + + +Future<bool> BindBackend::destroy(const string& rootfs) +{ + return dispatch(process.get(), &BindBackendProcess::destroy, rootfs); +} + + +Future<Nothing> BindBackendProcess::provision( + const vector<string>& layers, + const string& rootfs) +{ + if (layers.size() > 1) { + return Failure( + "Multiple layers are not supported by the bind backend"); + } + + if (layers.size() == 0) { + return Failure("No filesystem layer provided"); + } + + Try<Nothing> mkdir = os::mkdir(rootfs); + if (mkdir.isError()) { + return Failure("Failed to create container rootfs at " + rootfs); + } + + // TODO(xujyan): Use MS_REC? Does any provisioner use mounts within + // its image store in a single layer? + Try<Nothing> mount = fs::mount( + layers.front(), + rootfs, + None(), + MS_BIND, + NULL); + + if (mount.isError()) { + return Failure( + "Failed to bind mount rootfs '" + layers.front() + + "' to '" + rootfs + "': " + mount.error()); + } + + // And remount it read-only. + mount = fs::mount( + None(), // Ignored. + rootfs, + None(), + MS_BIND | MS_RDONLY | MS_REMOUNT, + NULL); + + if (mount.isError()) { + return Failure( + "Failed to remount rootfs '" + rootfs + "' read-only: " + + mount.error()); + } + + return Nothing(); +} + + +Future<bool> BindBackendProcess::destroy(const string& rootfs) +{ + Try<fs::MountInfoTable> mountTable = fs::MountInfoTable::read(); + + if (mountTable.isError()) { + return Failure("Failed to read mount table: " + mountTable.error()); + } + + foreach (const fs::MountInfoTable::Entry& entry, mountTable.get().entries) { + // TODO(xujyan): If MS_REC was used in 'provision()' we would need + // to check `strings::startsWith(entry.target, rootfs)` here to + // unmount all nested mounts. + if (entry.target == rootfs) { + // NOTE: This would fail if the rootfs is still in use. + Try<Nothing> unmount = fs::unmount(entry.target); + if (unmount.isError()) { + return Failure( + "Failed to destroy bind-mounted rootfs '" + rootfs + "': " + + unmount.error()); + } + + // TODO(jieyu): If 'rmdir' here returns EBUSY, we still returns + // a success. This is currently possible because the parent + // mount of 'rootfs' might not be a shared mount. Thus, + // containers in different mount namespaces might hold extra + // references to this mount. It is OK to ignore the EBUSY error + // because the provisioner will later try to delete all the + // rootfses for the terminated containers. + if (::rmdir(rootfs.c_str()) != 0) { + string message = + "Failed to remove rootfs mount point '" + rootfs + + "': " + strerror(errno); + + if (errno == EBUSY) { + LOG(ERROR) << message; + } else { + return Failure(message); + } + } + + return true; + } + } + + return false; +} + +} // namespace slave { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/backends/bind.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/backends/bind.hpp b/src/slave/containerizer/provisioner/backends/bind.hpp new file mode 100644 index 0000000..1685938 --- /dev/null +++ b/src/slave/containerizer/provisioner/backends/bind.hpp @@ -0,0 +1,75 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __PROVISIONER_BACKENDS_BIND_HPP__ +#define __PROVISIONER_BACKENDS_BIND_HPP__ + +#include "slave/containerizer/provisioner/backend.hpp" + +namespace mesos { +namespace internal { +namespace slave { + +// Forward declaration. +class BindBackendProcess; + + +// This is a specialized backend that may be useful for deployments +// using large (multi-GB) single-layer images *and* where more recent +// kernel features such as overlayfs are not available (overlayfs-based +// backend tracked by MESOS-2971). For small images (10's to 100's of MB) +// the copy backend may be sufficient. NOTE: +// 1) BindBackend supports only a single layer. Multi-layer images will +// fail to provision and the container will fail to launch! +// 2) The filesystem is read-only because all containers using this +// image share the source. Select writable areas can be achieved by +// mounting read-write volumes to places like /tmp, /var/tmp, +// /home, etc. using the ContainerInfo. These can be relative to +// the executor work directory. +// N.B. Since the filesystem is read-only, '--sandbox_directory' must +// already exist within the filesystem because the filesystem isolator +// is unable to create it! +// 3) It's fast because the bind mount requires (nearly) zero IO. +class BindBackend : public Backend +{ +public: + virtual ~BindBackend(); + + // BindBackend doesn't use any flag. + static Try<process::Owned<Backend>> create(const Flags&); + + virtual process::Future<Nothing> provision( + const std::vector<std::string>& layers, + const std::string& rootfs); + + virtual process::Future<bool> destroy(const std::string& rootfs); + +private: + explicit BindBackend(process::Owned<BindBackendProcess> process); + + BindBackend(const BindBackend&); // Not copyable. + BindBackend& operator=(const BindBackend&); // Not assignable. + + process::Owned<BindBackendProcess> process; +}; + +} // namespace slave { +} // namespace internal { +} // namespace mesos { + +#endif // __PROVISIONER_BACKENDS_BIND_HPP__ http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/backends/copy.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/backends/copy.cpp b/src/slave/containerizer/provisioner/backends/copy.cpp new file mode 100644 index 0000000..92fb098 --- /dev/null +++ b/src/slave/containerizer/provisioner/backends/copy.cpp @@ -0,0 +1,203 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include <list> + +#include <process/collect.hpp> +#include <process/defer.hpp> +#include <process/dispatch.hpp> +#include <process/io.hpp> +#include <process/process.hpp> +#include <process/subprocess.hpp> + + +#include <stout/foreach.hpp> +#include <stout/os.hpp> + +#include "common/status_utils.hpp" + +#include "slave/containerizer/provisioner/backends/copy.hpp" + + +using namespace process; + +using std::string; +using std::list; +using std::vector; + +namespace mesos { +namespace internal { +namespace slave { + +class CopyBackendProcess : public Process<CopyBackendProcess> +{ +public: + Future<Nothing> provision(const vector<string>& layers, const string& rootfs); + + Future<bool> destroy(const string& rootfs); + +private: + Future<Nothing> _provision(string layer, const string& rootfs); +}; + + +Try<Owned<Backend>> CopyBackend::create(const Flags&) +{ + return Owned<Backend>(new CopyBackend( + Owned<CopyBackendProcess>(new CopyBackendProcess()))); +} + + +CopyBackend::~CopyBackend() +{ + terminate(process.get()); + wait(process.get()); +} + + +CopyBackend::CopyBackend(Owned<CopyBackendProcess> _process) + : process(_process) +{ + spawn(CHECK_NOTNULL(process.get())); +} + + +Future<Nothing> CopyBackend::provision( + const vector<string>& layers, + const string& rootfs) +{ + return dispatch( + process.get(), &CopyBackendProcess::provision, layers, rootfs); +} + + +Future<bool> CopyBackend::destroy(const string& rootfs) +{ + return dispatch(process.get(), &CopyBackendProcess::destroy, rootfs); +} + + +Future<Nothing> CopyBackendProcess::provision( + const vector<string>& layers, + const string& rootfs) +{ + if (layers.size() == 0) { + return Failure("No filesystem layers provided"); + } + + if (os::exists(rootfs)) { + return Failure("Rootfs is already provisioned"); + } + + Try<Nothing> mkdir = os::mkdir(rootfs); + if (mkdir.isError()) { + return Failure("Failed to create rootfs directory: " + mkdir.error()); + } + + list<Future<Nothing>> futures{Nothing()}; + + foreach (const string layer, layers) { + futures.push_back( + futures.back().then( + defer(self(), &Self::_provision, layer, rootfs))); + } + + return collect(futures) + .then([]() -> Future<Nothing> { return Nothing(); }); +} + + +Future<Nothing> CopyBackendProcess::_provision( + string layer, + const string& rootfs) +{ + VLOG(1) << "Copying layer path '" << layer << "' to rootfs '" << rootfs + << "'"; + +#ifdef __APPLE__ + if (!strings::endsWith(layer, "/")) { + layer += "/"; + } + + // OSX cp doesn't support -T flag, but supports source trailing + // slash so we only copy the content but not the folder. + vector<string> args{"cp", "-a", layer, rootfs}; +#else + vector<string> args{"cp", "-aT", layer, rootfs}; +#endif // __APPLE__ + + Try<Subprocess> s = subprocess( + "cp", + args, + Subprocess::PATH("/dev/null"), + Subprocess::PATH("/dev/null"), + Subprocess::PIPE()); + + if (s.isError()) { + return Failure("Failed to create 'cp' subprocess: " + s.error()); + } + + Subprocess cp = s.get(); + + return cp.status() + .then([cp](const Option<int>& status) -> Future<Nothing> { + if (status.isNone()) { + return Failure("Failed to reap subprocess to copy image"); + } else if (status.get() != 0) { + return io::read(cp.err().get()) + .then([](const string& err) -> Future<Nothing> { + return Failure("Failed to copy layer: " + err); + }); + } + + return Nothing(); + }); +} + + +Future<bool> CopyBackendProcess::destroy(const string& rootfs) +{ + vector<string> argv{"rm", "-rf", rootfs}; + + Try<Subprocess> s = subprocess( + "rm", + argv, + Subprocess::PATH("/dev/null"), + Subprocess::FD(STDOUT_FILENO), + Subprocess::FD(STDERR_FILENO)); + + if (s.isError()) { + return Failure("Failed to create 'rm' subprocess: " + s.error()); + } + + return s.get().status() + .then([](const Option<int>& status) -> Future<bool> { + if (status.isNone()) { + return Failure("Failed to reap subprocess to destroy rootfs"); + } else if (status.get() != 0) { + return Failure("Failed to destroy rootfs, exit status: " + + WSTRINGIFY(status.get())); + } + + return true; + }); +} + +} // namespace slave { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/cc1f8f54/src/slave/containerizer/provisioner/backends/copy.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/provisioner/backends/copy.hpp b/src/slave/containerizer/provisioner/backends/copy.hpp new file mode 100644 index 0000000..10d9aee --- /dev/null +++ b/src/slave/containerizer/provisioner/backends/copy.hpp @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef __PROVISIONER_BACKENDS_COPY_HPP__ +#define __PROVISIONER_BACKENDS_COPY_HPP__ + +#include "slave/containerizer/provisioner/backend.hpp" + +namespace mesos { +namespace internal { +namespace slave { + +// Forward declaration. +class CopyBackendProcess; + + +class CopyBackend : public Backend +{ +public: + virtual ~CopyBackend(); + + // CopyBackend doesn't use any flag. + static Try<process::Owned<Backend>> create(const Flags&); + + // Provisions a rootfs given the layers' paths and target rootfs + // path. + virtual process::Future<Nothing> provision( + const std::vector<std::string>& layers, + const std::string& rootfs); + + virtual process::Future<bool> destroy(const std::string& rootfs); + +private: + explicit CopyBackend(process::Owned<CopyBackendProcess> process); + + CopyBackend(const CopyBackend&); // Not copyable. + CopyBackend& operator=(const CopyBackend&); // Not assignable. + + process::Owned<CopyBackendProcess> process; +}; + +} // namespace slave { +} // namespace internal { +} // namespace mesos { + +#endif // __PROVISIONER_BACKENDS_COPY_HPP__
