Repository: mesos Updated Branches: refs/heads/master 6195e783d -> 6afd1b682
Fixed support for non-HDFS URIs by Hadoop client. Review: https://reviews.apache.org/r/42246/ Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/6afd1b68 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/6afd1b68 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/6afd1b68 Branch: refs/heads/master Commit: 6afd1b682df9809a9113ab9f61caf26b8c6a02d9 Parents: 6195e78 Author: Bernd Mathiske <[email protected]> Authored: Fri Jan 22 11:36:12 2016 +0100 Committer: Bernd Mathiske <[email protected]> Committed: Fri Jan 22 11:36:37 2016 +0100 ---------------------------------------------------------------------- src/hdfs/hdfs.cpp | 36 ++++++++++++++++++++---------------- src/hdfs/hdfs.hpp | 4 ---- 2 files changed, 20 insertions(+), 20 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/6afd1b68/src/hdfs/hdfs.cpp ---------------------------------------------------------------------- diff --git a/src/hdfs/hdfs.cpp b/src/hdfs/hdfs.cpp index 9c91d1f..bebba8b 100644 --- a/src/hdfs/hdfs.cpp +++ b/src/hdfs/hdfs.cpp @@ -122,11 +122,26 @@ Try<Owned<HDFS>> HDFS::create(const Option<string>& _hadoop) } +// An HDFS client path must be either a full URI or an absolute path. If it is +// a relative path, prepend "/" to make it absolute. (Note that all URI schemes +// supported by the HDFS client contain "://" whereas file paths never do.) +static string normalize(const string& hdfsPath) +{ + if (strings::contains(hdfsPath, "://") || // A URI or a malformed path. + strings::startsWith(hdfsPath, "/")) { // Already an absolute path. + return hdfsPath; + } + + // A relative, non-URI file path. Prepend "/". + return path::join("", hdfsPath); +} + + Future<bool> HDFS::exists(const string& path) { Try<Subprocess> s = subprocess( hadoop, - {"hadoop", "fs", "-test", "-e", absolutePath(path)}, + {"hadoop", "fs", "-test", "-e", normalize(path)}, Subprocess::PATH("/dev/null"), Subprocess::PIPE(), Subprocess::PIPE()); @@ -161,7 +176,7 @@ Future<bool> HDFS::exists(const string& path) Future<Bytes> HDFS::du(const string& _path) { - const string path = absolutePath(_path); + const string path = normalize(_path); Try<Subprocess> s = subprocess( hadoop, @@ -218,7 +233,7 @@ Future<Nothing> HDFS::rm(const string& path) { Try<Subprocess> s = subprocess( hadoop, - {"hadoop", "fs", "-rm", absolutePath(path)}, + {"hadoop", "fs", "-rm", normalize(path)}, Subprocess::PATH("/dev/null"), Subprocess::PIPE(), Subprocess::PIPE()); @@ -254,7 +269,7 @@ Future<Nothing> HDFS::copyFromLocal(const string& from, const string& to) Try<Subprocess> s = subprocess( hadoop, - {"hadoop", "fs", "-copyFromLocal", from, absolutePath(to)}, + {"hadoop", "fs", "-copyFromLocal", from, normalize(to)}, Subprocess::PATH("/dev/null"), Subprocess::PIPE(), Subprocess::PIPE()); @@ -286,7 +301,7 @@ Future<Nothing> HDFS::copyToLocal(const string& from, const string& to) { Try<Subprocess> s = subprocess( hadoop, - {"hadoop", "fs", "-copyToLocal", absolutePath(from), to}, + {"hadoop", "fs", "-copyToLocal", normalize(from), to}, Subprocess::PATH("/dev/null"), Subprocess::PIPE(), Subprocess::PIPE()); @@ -312,14 +327,3 @@ Future<Nothing> HDFS::copyToLocal(const string& from, const string& to) return Nothing(); }); } - - -string HDFS::absolutePath(const string& hdfsPath) -{ - if (strings::startsWith(hdfsPath, "hdfs://") || - strings::startsWith(hdfsPath, "/")) { - return hdfsPath; - } - - return path::join("", hdfsPath); -} http://git-wip-us.apache.org/repos/asf/mesos/blob/6afd1b68/src/hdfs/hdfs.hpp ---------------------------------------------------------------------- diff --git a/src/hdfs/hdfs.hpp b/src/hdfs/hdfs.hpp index abdb9b9..716d13f 100644 --- a/src/hdfs/hdfs.hpp +++ b/src/hdfs/hdfs.hpp @@ -65,10 +65,6 @@ private: explicit HDFS(const std::string& _hadoop) : hadoop(_hadoop) {} - // Normalize an HDFS path such that it is either an absolute path or - // a full hdfs:// URL. - std::string absolutePath(const std::string& hdfsPath); - const std::string hadoop; };
