Added new parameter 'data' to the URI fetcher interface. Review: https://reviews.apache.org/r/59141
Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/92f16fc2 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/92f16fc2 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/92f16fc2 Branch: refs/heads/master Commit: 92f16fc22b78abc2f407d1730e901e6890d31bf5 Parents: 618a101 Author: Gilbert Song <songzihao1...@gmail.com> Authored: Wed May 10 04:45:53 2017 -0700 Committer: Gilbert Song <songzihao1...@gmail.com> Committed: Thu May 25 01:04:30 2017 +0800 ---------------------------------------------------------------------- include/mesos/uri/fetcher.hpp | 16 +++++++++++++--- .../mesos/provisioner/docker/registry_puller.cpp | 10 ++++++++-- src/tests/uri_fetcher_tests.cpp | 10 ++++++---- src/uri/fetcher.cpp | 10 ++++++---- src/uri/fetchers/copy.cpp | 3 ++- src/uri/fetchers/copy.hpp | 3 ++- src/uri/fetchers/curl.cpp | 3 ++- src/uri/fetchers/curl.hpp | 3 ++- src/uri/fetchers/docker.cpp | 8 ++++++-- src/uri/fetchers/docker.hpp | 3 ++- src/uri/fetchers/hadoop.cpp | 3 ++- src/uri/fetchers/hadoop.hpp | 3 ++- 12 files changed, 53 insertions(+), 22 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/92f16fc2/include/mesos/uri/fetcher.hpp ---------------------------------------------------------------------- diff --git a/include/mesos/uri/fetcher.hpp b/include/mesos/uri/fetcher.hpp index ebf86c7..760d6b3 100644 --- a/include/mesos/uri/fetcher.hpp +++ b/include/mesos/uri/fetcher.hpp @@ -21,6 +21,8 @@ #include <string> #include <vector> +#include <mesos/mesos.hpp> + #include <process/future.hpp> #include <process/owned.hpp> #include <process/shared.hpp> @@ -69,10 +71,14 @@ public: * * @param uri the URI to fetch * @param directory the directory the URI will be downloaded to + * @param data the optional user defined data */ + // TODO(gilbert): Change the parameter 'data' as a hashmap + // of <string, Secret::Value>, and update the comment. virtual process::Future<Nothing> fetch( const URI& uri, - const std::string& directory) const = 0; + const std::string& directory, + const Option<std::string>& data = None()) const = 0; }; /** @@ -88,11 +94,13 @@ public: * * @param uri the URI to fetch * @param directory the directory the URI will be downloaded to + * @param data the optional user defined data */ // TODO(jieyu): Consider using 'Path' for 'directory' here. process::Future<Nothing> fetch( const URI& uri, - const std::string& directory) const; + const std::string& directory, + const Option<std::string>& data = None()) const; /** * Fetches a URI to the given directory. This method will dispatch @@ -101,11 +109,13 @@ public: * @param uri the URI to fetch * @param directory the directory the URI will be downloaded to * @param name of the plugin that is used to download + * @param data the optional user defined data */ process::Future<Nothing> fetch( const URI& uri, const std::string& directory, - const std::string& name) const; + const std::string& name, + const Option<std::string>& data = None()) const; private: Fetcher(const Fetcher&) = delete; // Not copyable. http://git-wip-us.apache.org/repos/asf/mesos/blob/92f16fc2/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp ---------------------------------------------------------------------- diff --git a/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp index a5f299a..693e841 100644 --- a/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp +++ b/src/slave/containerizer/mesos/provisioner/docker/registry_puller.cpp @@ -288,7 +288,10 @@ Future<vector<string>> RegistryPullerProcess::_pull( << "' from '" << manifestUri << "' to '" << directory << "'"; - return fetcher->fetch(manifestUri, directory) + return fetcher->fetch( + manifestUri, + directory, + config.isSome() ? config->data() : Option<string>()) .then(defer(self(), &Self::__pull, reference, directory, backend, config)); } @@ -504,7 +507,10 @@ Future<hashset<string>> RegistryPullerProcess::fetchBlobs( port); } - futures.push_back(fetcher->fetch(blobUri, directory)); + futures.push_back(fetcher->fetch( + blobUri, + directory, + config.isSome() ? config->data() : Option<string>())); } return collect(futures) http://git-wip-us.apache.org/repos/asf/mesos/blob/92f16fc2/src/tests/uri_fetcher_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/uri_fetcher_tests.cpp b/src/tests/uri_fetcher_tests.cpp index 4892909..8a40f46 100644 --- a/src/tests/uri_fetcher_tests.cpp +++ b/src/tests/uri_fetcher_tests.cpp @@ -143,7 +143,7 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(CurlFetcherPluginTest, CURL_InvokeFetchByName) Try<Owned<uri::Fetcher>> fetcher = uri::fetcher::create(); ASSERT_SOME(fetcher); - AWAIT_READY(fetcher.get()->fetch(uri, os::getcwd(), "curl")); + AWAIT_READY(fetcher.get()->fetch(uri, os::getcwd(), "curl", None())); EXPECT_TRUE(os::exists(path::join(os::getcwd(), "test"))); } @@ -258,7 +258,7 @@ TEST_F(HadoopFetcherPluginTest, InvokeFetchByName) string dir = path::join(os::getcwd(), "dir"); - AWAIT_READY(fetcher.get()->fetch(uri, dir, "hadoop")); + AWAIT_READY(fetcher.get()->fetch(uri, dir, "hadoop", None())); EXPECT_SOME_EQ("abc", os::read(path::join(dir, "file"))); } @@ -364,7 +364,9 @@ TEST_F(DockerFetcherPluginTest, INTERNET_CURL_InvokeFetchByName) string dir = path::join(os::getcwd(), "dir"); - AWAIT_READY_FOR(fetcher.get()->fetch(uri, dir, "docker"), Seconds(60)); + AWAIT_READY_FOR( + fetcher.get()->fetch(uri, dir, "docker", None()), + Seconds(60)); Try<string> _manifest = os::read(path::join(dir, "manifest")); ASSERT_SOME(_manifest); @@ -440,7 +442,7 @@ TEST_F_TEMP_DISABLED_ON_WINDOWS(CopyFetcherPluginTest, InvokeFetchByName) const string dir = path::join(os::getcwd(), "dir"); - AWAIT_READY(fetcher.get()->fetch(uri, dir, "copy")); + AWAIT_READY(fetcher.get()->fetch(uri, dir, "copy", None())); // Validate the fetched file's content. EXPECT_SOME_EQ("abc", os::read(path::join(dir, "file"))); http://git-wip-us.apache.org/repos/asf/mesos/blob/92f16fc2/src/uri/fetcher.cpp ---------------------------------------------------------------------- diff --git a/src/uri/fetcher.cpp b/src/uri/fetcher.cpp index f4d1a4c..13c2d54 100644 --- a/src/uri/fetcher.cpp +++ b/src/uri/fetcher.cpp @@ -105,26 +105,28 @@ Fetcher::Fetcher(const vector<Owned<Plugin>>& plugins) Future<Nothing> Fetcher::fetch( const URI& uri, - const string& directory) const + const string& directory, + const Option<string>& data) const { if (!pluginsByScheme.contains(uri.scheme())) { return Failure("Scheme '" + uri.scheme() + "' is not supported"); } - return pluginsByScheme.at(uri.scheme())->fetch(uri, directory); + return pluginsByScheme.at(uri.scheme())->fetch(uri, directory, data); } Future<Nothing> Fetcher::fetch( const URI& uri, const string& directory, - const string& name) const + const string& name, + const Option<string>& data) const { if (!pluginsByName.contains(name)) { return Failure("Plugin '" + name + "' is not registered."); } - return pluginsByName.at(name)->fetch(uri, directory); + return pluginsByName.at(name)->fetch(uri, directory, data); } } // namespace uri { http://git-wip-us.apache.org/repos/asf/mesos/blob/92f16fc2/src/uri/fetchers/copy.cpp ---------------------------------------------------------------------- diff --git a/src/uri/fetchers/copy.cpp b/src/uri/fetchers/copy.cpp index 5e14705..17f69be 100644 --- a/src/uri/fetchers/copy.cpp +++ b/src/uri/fetchers/copy.cpp @@ -76,7 +76,8 @@ string CopyFetcherPlugin::name() const Future<Nothing> CopyFetcherPlugin::fetch( const URI& uri, - const string& directory) const + const string& directory, + const Option<string>& data) const { // TODO(jojy): Validate the given URI. http://git-wip-us.apache.org/repos/asf/mesos/blob/92f16fc2/src/uri/fetchers/copy.hpp ---------------------------------------------------------------------- diff --git a/src/uri/fetchers/copy.hpp b/src/uri/fetchers/copy.hpp index f4a2fb3..4d612fe 100644 --- a/src/uri/fetchers/copy.hpp +++ b/src/uri/fetchers/copy.hpp @@ -44,7 +44,8 @@ public: virtual process::Future<Nothing> fetch( const URI& uri, - const std::string& directory) const; + const std::string& directory, + const Option<std::string>& data = None()) const; private: CopyFetcherPlugin() {} http://git-wip-us.apache.org/repos/asf/mesos/blob/92f16fc2/src/uri/fetchers/curl.cpp ---------------------------------------------------------------------- diff --git a/src/uri/fetchers/curl.cpp b/src/uri/fetchers/curl.cpp index 24b53c7..f34daf2 100644 --- a/src/uri/fetchers/curl.cpp +++ b/src/uri/fetchers/curl.cpp @@ -79,7 +79,8 @@ string CurlFetcherPlugin::name() const Future<Nothing> CurlFetcherPlugin::fetch( const URI& uri, - const string& directory) const + const string& directory, + const Option<string>& data) const { // TODO(jieyu): Validate the given URI. http://git-wip-us.apache.org/repos/asf/mesos/blob/92f16fc2/src/uri/fetchers/curl.hpp ---------------------------------------------------------------------- diff --git a/src/uri/fetchers/curl.hpp b/src/uri/fetchers/curl.hpp index 083f155..909c2eb 100644 --- a/src/uri/fetchers/curl.hpp +++ b/src/uri/fetchers/curl.hpp @@ -44,7 +44,8 @@ public: virtual process::Future<Nothing> fetch( const URI& uri, - const std::string& directory) const; + const std::string& directory, + const Option<std::string>& data = None()) const; private: CurlFetcherPlugin() {} http://git-wip-us.apache.org/repos/asf/mesos/blob/92f16fc2/src/uri/fetchers/docker.cpp ---------------------------------------------------------------------- diff --git a/src/uri/fetchers/docker.cpp b/src/uri/fetchers/docker.cpp index 8aab169..6436f74 100644 --- a/src/uri/fetchers/docker.cpp +++ b/src/uri/fetchers/docker.cpp @@ -378,7 +378,10 @@ public: : ProcessBase(process::ID::generate("docker-fetcher-plugin")), auths(_auths) {} - Future<Nothing> fetch(const URI& uri, const string& directory); + Future<Nothing> fetch( + const URI& uri, + const string& directory, + const Option<string>& data); private: Future<Nothing> _fetch( @@ -489,7 +492,8 @@ string DockerFetcherPlugin::name() const Future<Nothing> DockerFetcherPlugin::fetch( const URI& uri, - const string& directory) const + const string& directory, + const Option<string>& data) const { return dispatch( process.get(), http://git-wip-us.apache.org/repos/asf/mesos/blob/92f16fc2/src/uri/fetchers/docker.hpp ---------------------------------------------------------------------- diff --git a/src/uri/fetchers/docker.hpp b/src/uri/fetchers/docker.hpp index 65e01cb..6a89c0a 100644 --- a/src/uri/fetchers/docker.hpp +++ b/src/uri/fetchers/docker.hpp @@ -54,7 +54,8 @@ public: virtual process::Future<Nothing> fetch( const URI& uri, - const std::string& directory) const; + const std::string& directory, + const Option<std::string>& data = None()) const; private: explicit DockerFetcherPlugin( http://git-wip-us.apache.org/repos/asf/mesos/blob/92f16fc2/src/uri/fetchers/hadoop.cpp ---------------------------------------------------------------------- diff --git a/src/uri/fetchers/hadoop.cpp b/src/uri/fetchers/hadoop.cpp index 3c5ffe6..2fa5b02 100644 --- a/src/uri/fetchers/hadoop.cpp +++ b/src/uri/fetchers/hadoop.cpp @@ -78,7 +78,8 @@ string HadoopFetcherPlugin::name() const Future<Nothing> HadoopFetcherPlugin::fetch( const URI& uri, - const string& directory) const + const string& directory, + const Option<string>& data) const { // TODO(jieyu): Validate the given URI. http://git-wip-us.apache.org/repos/asf/mesos/blob/92f16fc2/src/uri/fetchers/hadoop.hpp ---------------------------------------------------------------------- diff --git a/src/uri/fetchers/hadoop.hpp b/src/uri/fetchers/hadoop.hpp index 4923dc6..46f74c8 100644 --- a/src/uri/fetchers/hadoop.hpp +++ b/src/uri/fetchers/hadoop.hpp @@ -57,7 +57,8 @@ public: virtual process::Future<Nothing> fetch( const URI& uri, - const std::string& directory) const; + const std::string& directory, + const Option<std::string>& data = None()) const; private: HadoopFetcherPlugin(
