Introduced Appc image fetcher. Added implementation for simple image discovery for Appc images.
Review: https://reviews.apache.org/r/43336/ Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/4107f14e Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/4107f14e Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/4107f14e Branch: refs/heads/master Commit: 4107f14e3f2864e0840c6712207f5c49bb8cb6b4 Parents: 906566d Author: Jojy Varghese <[email protected]> Authored: Wed Feb 17 11:33:38 2016 -0800 Committer: Jie Yu <[email protected]> Committed: Wed Feb 17 12:01:53 2016 -0800 ---------------------------------------------------------------------- src/CMakeLists.txt | 1 + src/Makefile.am | 2 + .../mesos/provisioner/appc/fetcher.cpp | 228 +++++++++++++++++++ .../mesos/provisioner/appc/fetcher.hpp | 80 +++++++ src/slave/flags.cpp | 6 + src/slave/flags.hpp | 2 + 6 files changed, 319 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/4107f14e/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9ab84c0..5cf0ec8 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -244,6 +244,7 @@ if (NOT WIN32) slave/containerizer/mesos/provisioner/provisioner.cpp slave/containerizer/mesos/provisioner/store.cpp slave/containerizer/mesos/provisioner/appc/cache.cpp + slave/containerizer/mesos/provisioner/appc/fetcher.cpp slave/containerizer/mesos/provisioner/appc/paths.cpp slave/containerizer/mesos/provisioner/appc/store.cpp slave/containerizer/mesos/provisioner/backend.cpp http://git-wip-us.apache.org/repos/asf/mesos/blob/4107f14e/src/Makefile.am ---------------------------------------------------------------------- diff --git a/src/Makefile.am b/src/Makefile.am index 5813ab2..54ebe91 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -668,6 +668,7 @@ libmesos_no_3rdparty_la_SOURCES += \ slave/containerizer/mesos/provisioner/provisioner.cpp \ slave/containerizer/mesos/provisioner/store.cpp \ slave/containerizer/mesos/provisioner/appc/cache.cpp \ + slave/containerizer/mesos/provisioner/appc/fetcher.cpp \ slave/containerizer/mesos/provisioner/appc/paths.cpp \ slave/containerizer/mesos/provisioner/appc/store.cpp \ slave/containerizer/mesos/provisioner/backends/copy.cpp \ @@ -778,6 +779,7 @@ libmesos_no_3rdparty_la_SOURCES += \ slave/containerizer/mesos/provisioner/provisioner.hpp \ slave/containerizer/mesos/provisioner/store.hpp \ slave/containerizer/mesos/provisioner/appc/cache.hpp \ + slave/containerizer/mesos/provisioner/appc/fetcher.hpp \ slave/containerizer/mesos/provisioner/appc/paths.hpp \ slave/containerizer/mesos/provisioner/appc/store.hpp \ slave/containerizer/mesos/provisioner/backends/copy.hpp \ http://git-wip-us.apache.org/repos/asf/mesos/blob/4107f14e/src/slave/containerizer/mesos/provisioner/appc/fetcher.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/appc/fetcher.cpp b/src/slave/containerizer/mesos/provisioner/appc/fetcher.cpp new file mode 100644 index 0000000..e12a6f2 --- /dev/null +++ b/src/slave/containerizer/mesos/provisioner/appc/fetcher.cpp @@ -0,0 +1,228 @@ +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <stout/os.hpp> +#include <stout/strings.hpp> + +#include <mesos/mesos.hpp> + +#include "common/command_utils.hpp" + +#include "slave/containerizer/mesos/provisioner/appc/fetcher.hpp" +#include "slave/containerizer/mesos/provisioner/appc/paths.hpp" + +#include "uri/schemes/http.hpp" + +namespace http = process::http; + +using std::string; + +using process::Failure; +using process::Future; +using process::Owned; +using process::Shared; + +namespace mesos { +namespace internal { +namespace slave { +namespace appc { + +static const char EXT[] = "aci"; +static const char LABEL_VERSION[] = "version"; +static const char LABEL_OS[] = "os"; +static const char LABEL_ARCH[] = "arch"; + + +static Try<string> getSimpleDiscoveryImagePath(const Image::Appc& appc) +{ + CHECK(!appc.name().empty()); + + hashmap<string, string> labels; + foreach (const mesos::Label& label, appc.labels().labels()) { + labels[label.key()] = label.value(); + } + + if (!labels.contains(LABEL_VERSION)) { + labels.insert({LABEL_VERSION, "latest"}); + } + + if (!labels.contains(LABEL_OS)) { + return Error( + "Failed to form simple discovery url: label '" + + string(LABEL_OS) + "' is missing"); + } + + if (!labels.contains(LABEL_ARCH)) { + return Error( + "Failed to form simple discovery url: label '" + + string(LABEL_ARCH) + "' is missing"); + } + + return strings::format( + "%s-%s-%s-%s.%s", + appc.name(), // Image name. + labels.at(LABEL_VERSION), // Version label. + labels.at(LABEL_OS), // OS label. + labels.at(LABEL_ARCH), // ARCH label. + EXT); // Extension. +} + + +static Try<URI> getUri(const string& prefix, const string& path) +{ + const string rawUrl = prefix + path; + + // TODO(jojy): Add parse URI function in URI namespace. + Try<http::URL> _url = http::URL::parse(rawUrl); + if (_url.isError()) { + return Error( + "Failed to parse '" + rawUrl + "' as a valid URL: " + _url.error()); + } + + const http::URL& url = _url.get(); + + if (url.domain.isNone() && url.ip.isNone()) { + return Error( + "Failed to parse host name from image url '" + rawUrl + "'"); + } + + // Get image server port. + if (url.port.isNone()) { + return Error( + "Failed to parse port for image url '" + rawUrl + "'"); + } + + // Get image server host. + const string host = url.domain.isSome() + ? url.domain.get() + : stringify(url.ip.get()); + + int port = static_cast<int>(url.port.get()); + + if (url.scheme.get() == "http") { + return uri::http( host, url.path, port); + } + + if (url.scheme.get() == "https") { + return uri::https(host, url.path, port); + } + + // TODO(jojy): Add support for hdfs. + + return Error("Unsupported scheme '" + url.scheme.get() + "'"); +} + + +Try<Owned<Fetcher>> Fetcher::create( + const Flags& flags, + const Shared<uri::Fetcher>& fetcher) +{ + const string prefix = flags.appc_simple_discovery_uri_prefix; + + // TODO(jojy): Add support for hdfs. + if (!strings::startsWith(prefix, "http") && + !strings::startsWith(prefix, "https")) { + return Error("Invalid simple discovery uri prefix: " + prefix); + } + + return new Fetcher(prefix, fetcher); +} + + +Fetcher::Fetcher( + const string& _uriPrefix, + const Shared<uri::Fetcher>& _fetcher) + : uriPrefix(_uriPrefix), + fetcher(_fetcher) {} + + +Future<Nothing> Fetcher::fetch(const Image::Appc& appc, const Path& directory) +{ + // TODO(jojy): Add more discovery implementations. We use simple + // discovery now but in the future, this will be extended to use + // multiple discoveries and try sequentially based on priority. + + if (appc.name().empty()) { + return Failure("Image name cannot be empty"); + } + + Try<string> path = getSimpleDiscoveryImagePath(appc); + if (path.isError()) { + return Failure( + "Failed to get discovery path for image '" + + appc.name() + "': " + path.error()); + } + + // First construct a URI based on the scheme. + Try<URI> uri = getUri(uriPrefix, path.get()); + if (uri.isError()) { + return Failure( + "Failed to get URI for image discovery path '" + + path.get() + "': " + uri.error()); + } + + VLOG(1) << "Fetching image from URI '" << uri.get() << "'"; + + // NOTE: URI fetcher will fetch the image into 'directory' with file + // name as URI's basename. + const Path aciBundle(path::join( + directory, + Path(uri->path()).basename())); + + return fetcher->fetch(uri.get(), directory) + .then([=]() -> Future<Nothing> { + // Change the extension to ".gz" as gzip utility expects it. + const Path _aciBundle(aciBundle.value + ".gz"); + + Try<Nothing> rename = os::rename(aciBundle, _aciBundle); + if (rename.isError()) { + return Failure( + "Failed to change extension to 'gz' for bundle '" + + stringify(aciBundle) + "': " + rename.error()); + } + + return command::decompress(_aciBundle); + }) + .then([=]() -> Future<string> { + return command::sha512(aciBundle); + }) + .then([=](const string& shasum) -> Future<Nothing> { + const string imagePath(path::join(directory, "sha512-" + shasum)); + + Try<Nothing> mkdir = os::mkdir(imagePath); + if (mkdir.isError()) { + return Failure( + "Failed to create directory for untarring image '" + + appc.name() + "': " + mkdir.error()); + } + + return command::untar(aciBundle, imagePath); + }) + .then([=]() -> Future<Nothing> { + // Remove the bundle file if everything goes well. + Try<Nothing> remove = os::rm(aciBundle); + if (remove.isError()) { + return Failure( + "Failed to remove aci bundle file '" + stringify(aciBundle) + + "': " + remove.error()); + } + + return Nothing(); + }); +} + +} // namespace appc { +} // namespace slave { +} // namespace internal { +} // namespace mesos { http://git-wip-us.apache.org/repos/asf/mesos/blob/4107f14e/src/slave/containerizer/mesos/provisioner/appc/fetcher.hpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/appc/fetcher.hpp b/src/slave/containerizer/mesos/provisioner/appc/fetcher.hpp new file mode 100644 index 0000000..373c7d4 --- /dev/null +++ b/src/slave/containerizer/mesos/provisioner/appc/fetcher.hpp @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef __PROVISIONER_APPC_FETCHER_HPP__ +#define __PROVISIONER_APPC_FETCHER_HPP__ + +#include <string> + +#include <process/process.hpp> +#include <process/shared.hpp> + +#include <stout/path.hpp> + +#include "slave/flags.hpp" + +#include "uri/fetcher.hpp" + +namespace mesos { +namespace internal { +namespace slave { +namespace appc { + +class Fetcher +{ +public: + /** + * Factory method for creating the fetcher component. + * + * @param flags Slave flags. + * @param fetcher Shared pointer to the uri fetcher. + */ + static Try<process::Owned<Fetcher>> create( + const Flags& flags, + const process::Shared<uri::Fetcher>& fetcher); + + /* + * Fetches Appc image to the given directory. + * + * Reference: https://github.com/appc/spec/blob/master/spec/discovery.md + * + * @param image Encapsulated information about the appc image. + * @param directory Path of directory where the image has to be saved. + * @returns Nothing on success. + * Failure in case of any error. + */ + process::Future<Nothing> fetch( + const Image::Appc& appc, + const Path& directory); + +private: + Fetcher( + const std::string& uriPrefix, + const process::Shared<uri::Fetcher>& fetcher); + + Fetcher(const Fetcher&) = delete; + Fetcher& operator=(const Fetcher&) = delete; + + const std::string uriPrefix; + process::Shared<uri::Fetcher> fetcher; +}; + +} // namespace appc { +} // namespace slave { +} // namespace internal { +} // namespace mesos { + +#endif // __PROVISIONER_APPC_FETCHER_HPP__ http://git-wip-us.apache.org/repos/asf/mesos/blob/4107f14e/src/slave/flags.cpp ---------------------------------------------------------------------- diff --git a/src/slave/flags.cpp b/src/slave/flags.cpp index 46ccf7f..d4b4e52 100644 --- a/src/slave/flags.cpp +++ b/src/slave/flags.cpp @@ -113,6 +113,12 @@ mesos::internal::slave::Flags::Flags() "e.g., `bind`, `copy`.", "copy"); + add(&Flags::appc_simple_discovery_uri_prefix, + "appc_simple_discovery_uri_prefix", + "URI prefix to be used for simple discovery of appc images,\n" + "e.g., 'http://', 'https://', 'hdfs://<hostname>:9000/user/abc/cde'.", + "http://";); + add(&Flags::appc_store_dir, "appc_store_dir", "Directory the appc provisioner will store images in.\n", http://git-wip-us.apache.org/repos/asf/mesos/blob/4107f14e/src/slave/flags.hpp ---------------------------------------------------------------------- diff --git a/src/slave/flags.hpp b/src/slave/flags.hpp index 3704376..bd52b4f 100644 --- a/src/slave/flags.hpp +++ b/src/slave/flags.hpp @@ -49,6 +49,8 @@ public: Option<std::string> image_providers; std::string image_provisioner_backend; + + std::string appc_simple_discovery_uri_prefix; std::string appc_store_dir; std::string docker_auth_server;
