Repository: mesos Updated Branches: refs/heads/master e36e3b998 -> 1e7120833
Windows: Enabled `CurlFetcherPlugin` and `HadoopFetcherPlugin`. Replaced checking for `/` with `path::absolute`, and fixed `argv[0]` assumption when invoking `hadoop`. Ported the HDFS mock script from Shell to Batch to emulate the `hadoop` command. Enabled the plugins and associated tests. Review: https://reviews.apache.org/r/65617/ Review: https://reviews.apache.org/r/65618/ Review: https://reviews.apache.org/r/65619/ Review: https://reviews.apache.org/r/65620/ Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/1e712083 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/1e712083 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/1e712083 Branch: refs/heads/master Commit: 1e71208338fc525d42cd8dd0ffad5a0841e40a92 Parents: e36e3b9 Author: Jeff Coffler <j...@taltos.com> Authored: Wed Feb 14 12:33:20 2018 -0800 Committer: Andrew Schwartzmeyer <and...@schwartzmeyer.com> Committed: Wed Feb 14 14:01:51 2018 -0800 ---------------------------------------------------------------------- src/CMakeLists.txt | 6 +++--- src/hdfs/hdfs.cpp | 4 ++-- src/tests/uri_fetcher_tests.cpp | 35 ++++++++++++++++++++--------------- src/uri/fetcher.cpp | 4 ++-- src/uri/fetcher.hpp | 6 +++--- 5 files changed, 30 insertions(+), 25 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/1e712083/src/CMakeLists.txt ---------------------------------------------------------------------- diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 21fb47e..0c13503 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -373,12 +373,12 @@ set(URI_SRC uri/fetcher.cpp uri/utils.cpp uri/fetchers/copy.cpp - uri/fetchers/curl.cpp) + uri/fetchers/curl.cpp + uri/fetchers/hadoop.cpp) if (NOT WIN32) list(APPEND URI_SRC - uri/fetchers/docker.cpp - uri/fetchers/hadoop.cpp) + uri/fetchers/docker.cpp) endif () set(USAGE_SRC http://git-wip-us.apache.org/repos/asf/mesos/blob/1e712083/src/hdfs/hdfs.cpp ---------------------------------------------------------------------- diff --git a/src/hdfs/hdfs.cpp b/src/hdfs/hdfs.cpp index f9fc1cbb5..726925f 100644 --- a/src/hdfs/hdfs.cpp +++ b/src/hdfs/hdfs.cpp @@ -143,7 +143,7 @@ Try<Owned<HDFS>> HDFS::create(const Option<string>& _hadoop) static string normalize(const string& hdfsPath) { if (strings::contains(hdfsPath, "://") || // A URI or a malformed path. - strings::startsWith(hdfsPath, "/")) { // Already an absolute path. + path::absolute(hdfsPath)) { // Already an absolute path. return hdfsPath; } @@ -316,7 +316,7 @@ Future<Nothing> HDFS::copyToLocal(const string& from, const string& to) { Try<Subprocess> s = subprocess( hadoop, - {"hadoop", "fs", "-copyToLocal", normalize(from), to}, + {hadoop, "fs", "-copyToLocal", normalize(from), to}, Subprocess::PATH(os::DEV_NULL), Subprocess::PIPE(), Subprocess::PIPE()); http://git-wip-us.apache.org/repos/asf/mesos/blob/1e712083/src/tests/uri_fetcher_tests.cpp ---------------------------------------------------------------------- diff --git a/src/tests/uri_fetcher_tests.cpp b/src/tests/uri_fetcher_tests.cpp index 0acb854..816d8ec 100644 --- a/src/tests/uri_fetcher_tests.cpp +++ b/src/tests/uri_fetcher_tests.cpp @@ -31,6 +31,7 @@ #include <stout/os/getcwd.hpp> #include <stout/os/ls.hpp> #include <stout/os/write.hpp> +#include <stout/uri.hpp> #include <stout/tests/utils.hpp> @@ -158,12 +159,17 @@ public: // Create a fake hadoop command line tool. It emulates the hadoop // client's logic while operating on the local filesystem. +#ifndef __WINDOWS__ hadoop = path::join(os::getcwd(), "hadoop"); +#else + hadoop = path::join(os::getcwd(), "hadoop.bat"); +#endif // __WINDOWS__ // The script emulating 'hadoop fs -copyToLocal <from> <to>'. // NOTE: We emulate a version call here which is exercised when // creating the HDFS client. But, we don't expect any other // command to be called. +#ifndef __WINDOWS__ ASSERT_SOME(os::write( hadoop, "#!/bin/sh\n" @@ -178,12 +184,23 @@ public: "fi\n" "cp $3 $4\n")); - // Make sure the script has execution permission. On Windows, we always - // have permission. -#ifndef __WINDOWS__ + // Make sure the script has execution permission. ASSERT_SOME(os::chmod( hadoop, S_IRWXU | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)); +#else + // NOTE: This is a `.bat` file instead of PowerShell so that it can be + // directly executed. Windows "knows" how to launch files ending in `.bat`, + // similar to the shebang logic for POSIX. However, this does not extend to + // PowerShell's `.ps1` scripts. + ASSERT_SOME(os::write( + hadoop, + "if \"%1\" == \"version\" (exit 0)\n" + "if NOT \"%1\" == \"fs\" (exit 1)\n" + "if NOT \"%2\" == \"-copyToLocal\" (exit 1)\n" + "copy %3 %4\n")); + + // Windows has no notion of "execution permission". #endif // __WINDOWS__ } @@ -192,9 +209,6 @@ protected: }; -// TODO(hausdorff): Will not compile until HDFS is supported on Windows. See -// MESOS-5460. -#ifndef __WINDOWS__ TEST_F(HadoopFetcherPluginTest, FetchExistingFile) { string file = path::join(os::getcwd(), "file"); @@ -215,12 +229,8 @@ TEST_F(HadoopFetcherPluginTest, FetchExistingFile) EXPECT_SOME_EQ("abc", os::read(path::join(dir, "file"))); } -#endif // __WINDOWS__ -// TODO(hausdorff): Will not compile until HDFS is supported on Windows. See -// MESOS-5460. -#ifndef __WINDOWS__ TEST_F(HadoopFetcherPluginTest, FetchNonExistingFile) { URI uri = uri::hdfs(path::join(os::getcwd(), "non-exist")); @@ -235,12 +245,8 @@ TEST_F(HadoopFetcherPluginTest, FetchNonExistingFile) AWAIT_FAILED(fetcher.get()->fetch(uri, dir)); } -#endif // __WINDOWS__ -// TODO(hausdorff): Will not compile until HDFS is supported on Windows. See -// MESOS-5460. -#ifndef __WINDOWS__ // This test verifies invoking 'fetch' by plugin name. TEST_F(HadoopFetcherPluginTest, InvokeFetchByName) { @@ -262,7 +268,6 @@ TEST_F(HadoopFetcherPluginTest, InvokeFetchByName) EXPECT_SOME_EQ("abc", os::read(path::join(dir, "file"))); } -#endif // __WINDOWS__ // TODO(jieyu): Expose this constant so that other docker related http://git-wip-us.apache.org/repos/asf/mesos/blob/1e712083/src/uri/fetcher.cpp ---------------------------------------------------------------------- diff --git a/src/uri/fetcher.cpp b/src/uri/fetcher.cpp index 13c2d54..489c7ce 100644 --- a/src/uri/fetcher.cpp +++ b/src/uri/fetcher.cpp @@ -50,10 +50,10 @@ Try<Owned<Fetcher>> create(const Option<Flags>& _flags) [flags]() { return CurlFetcherPlugin::create(flags); }}, {CopyFetcherPlugin::NAME, [flags]() { return CopyFetcherPlugin::create(flags); }}, -#ifndef __WINDOWS__ - // TODO(dpravat): Enable `Hadoop` and `Docker` plugins. See MESOS-5473. {HadoopFetcherPlugin::NAME, [flags]() { return HadoopFetcherPlugin::create(flags); }}, +#ifndef __WINDOWS__ + // TODO(coffler): Enable Docker plugin. See MESOS-8570. {DockerFetcherPlugin::NAME, [flags]() { return DockerFetcherPlugin::create(flags); }}, #endif // __WINDOWS__ http://git-wip-us.apache.org/repos/asf/mesos/blob/1e712083/src/uri/fetcher.hpp ---------------------------------------------------------------------- diff --git a/src/uri/fetcher.hpp b/src/uri/fetcher.hpp index 2b2b14e..fbf64c6 100644 --- a/src/uri/fetcher.hpp +++ b/src/uri/fetcher.hpp @@ -39,11 +39,11 @@ namespace fetcher { */ class Flags : public virtual CopyFetcherPlugin::Flags, + public virtual CurlFetcherPlugin::Flags, #ifdef __WINDOWS__ - // TODO(dpravat): Add support for Hadoop and Docker plugins. See MESOS-5473. - public virtual CurlFetcherPlugin::Flags {}; + public virtual HadoopFetcherPlugin::Flags {}; #else - public virtual CurlFetcherPlugin::Flags, + // TODO(coffler): Add support for Docker plugins. See MESOS-8570. public virtual HadoopFetcherPlugin::Flags, public virtual DockerFetcherPlugin::Flags {}; #endif // __WINDOWS__
