Repository: mesos Updated Branches: refs/heads/master 4f780f939 -> 870dde216
Allow hdfs URLs in the HDFS wrapper API. Review: https://reviews.apache.org/r/39799 Project: http://git-wip-us.apache.org/repos/asf/mesos/repo Commit: http://git-wip-us.apache.org/repos/asf/mesos/commit/870dde21 Tree: http://git-wip-us.apache.org/repos/asf/mesos/tree/870dde21 Diff: http://git-wip-us.apache.org/repos/asf/mesos/diff/870dde21 Branch: refs/heads/master Commit: 870dde216bfe6f959654c4cb9f72d3876f9e2d56 Parents: 4f780f9 Author: James Peach <[email protected]> Authored: Wed Nov 4 13:07:00 2015 +0100 Committer: Bernd Mathiske <[email protected]> Committed: Wed Nov 4 13:09:43 2015 +0100 ---------------------------------------------------------------------- src/hdfs/hdfs.hpp | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/mesos/blob/870dde21/src/hdfs/hdfs.hpp ---------------------------------------------------------------------- diff --git a/src/hdfs/hdfs.hpp b/src/hdfs/hdfs.hpp index 18f1723..42c1501 100644 --- a/src/hdfs/hdfs.hpp +++ b/src/hdfs/hdfs.hpp @@ -83,8 +83,7 @@ struct HDFS Try<bool> exists(std::string path) { - // Make sure 'path' starts with a '/'. - path = path::join("", path); + path = absolutePath(path); Try<std::string> command = strings::format( "%s fs -test -e '%s'", hadoop, path); @@ -104,8 +103,7 @@ struct HDFS Try<Bytes> du(std::string path) { - // Make sure 'path' starts with a '/'. - path = path::join("", path); + path = absolutePath(path); Try<std::string> command = strings::format( "%s fs -du -h '%s'", hadoop, path); @@ -141,8 +139,7 @@ struct HDFS Try<Nothing> rm(std::string path) { - // Make sure 'to' starts with a '/'. - path = path::join("", path); + path = absolutePath(path); Try<std::string> command = strings::format( "%s fs -rm '%s'", hadoop, path); @@ -166,8 +163,7 @@ struct HDFS return Error("Failed to find " + from); } - // Make sure 'to' starts with a '/'. - to = path::join("", to); + to = absolutePath(to); // Copy to HDFS. Try<std::string> command = strings::format( @@ -185,9 +181,11 @@ struct HDFS } Try<Nothing> copyToLocal( - const std::string& from, + std::string from, const std::string& to) { + from = absolutePath(from); + // Copy from HDFS. Try<std::string> command = strings::format( "%s fs -copyToLocal '%s' '%s'", hadoop, from, to); @@ -204,6 +202,18 @@ struct HDFS } private: + // Normalize an HDFS path such that it is either an absolute path + // or a full hdfs:// URL. + std::string absolutePath(const std::string& hdfsPath) + { + if (strings::startsWith(hdfsPath, "hdfs://") || + strings::startsWith(hdfsPath, "/")) { + return hdfsPath; + } + + return path::join("", hdfsPath); + } + const std::string hadoop; };
