Add some path / env related helper functions * path_util: Add SplitPath() helper function
This function provides a (theoretically) portable method of splitting a filesystem path into its constituent components. * env: Add GetCurrentWorkingDir() and ChangeDir() These are functions that were missing from our Env implementation. * env_util: Implement CreateDirsRecursively() This function is similar in spirit to the "mkdir -p" command. Also reordered a few includes and deleted a few lines of commented-out code. Change-Id: Ia664708a09493923abbf2ff4e56e3d49c62cf97e Reviewed-on: http://gerrit.cloudera.org:8080/5618 Reviewed-by: Adar Dembo <[email protected]> Tested-by: Mike Percy <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/8598bd94 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/8598bd94 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/8598bd94 Branch: refs/heads/master Commit: 8598bd94779aa9da966b5b4cb5e613db46a60b5a Parents: 7679f9b Author: Mike Percy <[email protected]> Authored: Thu Jan 5 15:54:01 2017 -0800 Committer: Mike Percy <[email protected]> Committed: Thu Jan 19 19:09:15 2017 +0000 ---------------------------------------------------------------------- src/kudu/util/env-test.cc | 18 ++++++++++ src/kudu/util/env.h | 6 ++++ src/kudu/util/env_posix.cc | 21 +++++++++++ src/kudu/util/env_util-test.cc | 69 ++++++++++++++++++++++++++++++++---- src/kudu/util/env_util.cc | 32 ++++++++++++++--- src/kudu/util/env_util.h | 6 ++++ src/kudu/util/path_util-test.cc | 16 +++++++++ src/kudu/util/path_util.cc | 15 ++++++++ src/kudu/util/path_util.h | 4 +++ 9 files changed, 176 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/env-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/env-test.cc b/src/kudu/util/env-test.cc index a90f0e1..01e0db6 100644 --- a/src/kudu/util/env-test.cc +++ b/src/kudu/util/env-test.cc @@ -826,4 +826,22 @@ TEST_F(TestEnv, TestGetBytesFree) { << "Failed after " << kIters << " attempts"; } +TEST_F(TestEnv, TestChangeDir) { + string orig_dir; + ASSERT_OK(env_->GetCurrentWorkingDir(&orig_dir)); + + string cwd; + ASSERT_OK(env_->ChangeDir("/")); + ASSERT_OK(env_->GetCurrentWorkingDir(&cwd)); + ASSERT_EQ("/", cwd); + + ASSERT_OK(env_->ChangeDir(test_dir_)); + ASSERT_OK(env_->GetCurrentWorkingDir(&cwd)); + ASSERT_EQ(test_dir_, cwd); + + ASSERT_OK(env_->ChangeDir(orig_dir)); + ASSERT_OK(env_->GetCurrentWorkingDir(&cwd)); + ASSERT_EQ(orig_dir, cwd); +} + } // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/env.h ---------------------------------------------------------------------- diff --git a/src/kudu/util/env.h b/src/kudu/util/env.h index e585b85..78d4bb5 100644 --- a/src/kudu/util/env.h +++ b/src/kudu/util/env.h @@ -152,6 +152,12 @@ class Env { // Delete the specified directory. virtual Status DeleteDir(const std::string& dirname) = 0; + // Return the current working directory. + virtual Status GetCurrentWorkingDir(std::string* cwd) const = 0; + + // Change the current working directory. + virtual Status ChangeDir(const std::string& dest) = 0; + // Synchronize the entry for a specific directory. virtual Status SyncDir(const std::string& dirname) = 0; http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/env_posix.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/env_posix.cc b/src/kudu/util/env_posix.cc index 2516f61..2a83fa8 100644 --- a/src/kudu/util/env_posix.cc +++ b/src/kudu/util/env_posix.cc @@ -890,6 +890,27 @@ class PosixEnv : public Env { return result; }; + Status GetCurrentWorkingDir(string* cwd) const override { + TRACE_EVENT0("io", "PosixEnv::GetCurrentWorkingDir"); + ThreadRestrictions::AssertIOAllowed(); + unique_ptr<char, FreeDeleter> wd(getcwd(NULL, 0)); + if (!wd) { + return IOError("getcwd()", errno); + } + cwd->assign(wd.get()); + return Status::OK(); + } + + Status ChangeDir(const string& dest) override { + TRACE_EVENT1("io", "PosixEnv::ChangeDir", "dest", dest); + ThreadRestrictions::AssertIOAllowed(); + Status result; + if (chdir(dest.c_str()) != 0) { + result = IOError(dest, errno); + } + return result; + } + virtual Status SyncDir(const std::string& dirname) OVERRIDE { TRACE_EVENT1("io", "SyncDir", "path", dirname); ThreadRestrictions::AssertIOAllowed(); http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/env_util-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/env_util-test.cc b/src/kudu/util/env_util-test.cc index 3676d64..90b308d 100644 --- a/src/kudu/util/env_util-test.cc +++ b/src/kudu/util/env_util-test.cc @@ -15,41 +15,98 @@ // specific language governing permissions and limitations // under the License. +#include <unistd.h> + #include "kudu/util/env_util.h" #include <gflags/gflags.h> #include <memory> #include <sys/statvfs.h> +#include "kudu/gutil/strings/substitute.h" +#include "kudu/util/path_util.h" +#include "kudu/util/test_macros.h" #include "kudu/util/test_util.h" DECLARE_int64(disk_reserved_bytes); DECLARE_int64(disk_reserved_bytes_free_for_testing); -namespace kudu { - using std::string; using std::unique_ptr; +using strings::Substitute; + +namespace kudu { +namespace env_util { class EnvUtilTest: public KuduTest { }; TEST_F(EnvUtilTest, TestDiskSpaceCheck) { - Env* env = Env::Default(); - const int64_t kRequestedBytes = 0; int64_t reserved_bytes = 0; - ASSERT_OK(env_util::VerifySufficientDiskSpace(env, test_dir_, kRequestedBytes, reserved_bytes)); + ASSERT_OK(VerifySufficientDiskSpace(env_, test_dir_, kRequestedBytes, reserved_bytes)); // Make it seem as if the disk is full and specify that we should have // reserved 200 bytes. Even asking for 0 bytes should return an error // indicating we are out of space. FLAGS_disk_reserved_bytes_free_for_testing = 0; reserved_bytes = 200; - Status s = env_util::VerifySufficientDiskSpace(env, test_dir_, kRequestedBytes, reserved_bytes); + Status s = VerifySufficientDiskSpace(env_, test_dir_, kRequestedBytes, reserved_bytes); ASSERT_TRUE(s.IsIOError()); ASSERT_EQ(ENOSPC, s.posix_code()); ASSERT_STR_CONTAINS(s.ToString(), "Insufficient disk space"); } +// Ensure that we can recursively create directories using both absolute and +// relative paths. +TEST_F(EnvUtilTest, TestCreateDirsRecursively) { + // Absolute path. + string path = JoinPathSegments(test_dir_, "a/b/c"); + ASSERT_OK(CreateDirsRecursively(env_, path)); + bool is_dir; + ASSERT_OK(env_->IsDirectory(path, &is_dir)); + ASSERT_TRUE(is_dir); + + // Repeating the previous command should also succeed (it should be a no-op). + ASSERT_OK(CreateDirsRecursively(env_, path)); + ASSERT_OK(env_->IsDirectory(path, &is_dir)); + ASSERT_TRUE(is_dir); + + // Relative path. + ASSERT_OK(env_->ChangeDir(test_dir_)); // Change to test dir to keep CWD clean. + string rel_base = Substitute("$0-$1", CURRENT_TEST_CASE_NAME(), CURRENT_TEST_NAME()); + ASSERT_FALSE(env_->FileExists(rel_base)); + path = JoinPathSegments(rel_base, "x/y/z"); + ASSERT_OK(CreateDirsRecursively(env_, path)); + ASSERT_OK(env_->IsDirectory(path, &is_dir)); + ASSERT_TRUE(is_dir); + + // Directory creation should fail if a file is a part of the path. + path = JoinPathSegments(test_dir_, "x/y/z"); + string file_path = JoinPathSegments(test_dir_, "x"); // Conflicts with 'path'. + ASSERT_FALSE(env_->FileExists(path)); + ASSERT_FALSE(env_->FileExists(file_path)); + // Create an empty file in the path. + unique_ptr<WritableFile> out; + ASSERT_OK(env_->NewWritableFile(file_path, &out)); + ASSERT_OK(out->Close()); + ASSERT_TRUE(env_->FileExists(file_path)); + // Fail. + Status s = CreateDirsRecursively(env_, path); + ASSERT_TRUE(s.IsAlreadyPresent()) << s.ToString(); + ASSERT_STR_CONTAINS(s.ToString(), "File exists"); + + // We should be able to create a directory tree even when a symlink exists as + // part of the path. + path = JoinPathSegments(test_dir_, "link/a/b"); + string link_path = JoinPathSegments(test_dir_, "link"); + string real_dir = JoinPathSegments(test_dir_, "real_dir"); + ASSERT_OK(env_->CreateDir(real_dir)); + PCHECK(symlink(real_dir.c_str(), link_path.c_str()) == 0); + ASSERT_OK(CreateDirsRecursively(env_, path)); + ASSERT_OK(env_->IsDirectory(path, &is_dir)); + ASSERT_TRUE(is_dir); +} + +} // namespace env_util } // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/env_util.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/env_util.cc b/src/kudu/util/env_util.cc index 27cdbba..e398831 100644 --- a/src/kudu/util/env_util.cc +++ b/src/kudu/util/env_util.cc @@ -15,10 +15,13 @@ // specific language governing permissions and limitations // under the License. +#include "kudu/util/env_util.h" + #include <algorithm> #include <memory> #include <string> #include <utility> +#include <vector> #include <gflags/gflags.h> #include <glog/logging.h> @@ -30,9 +33,9 @@ #include "kudu/gutil/strings/util.h" #include "kudu/util/debug-util.h" #include "kudu/util/env.h" -#include "kudu/util/env_util.h" -#include "kudu/util/status.h" #include "kudu/util/flag_tags.h" +#include "kudu/util/path_util.h" +#include "kudu/util/status.h" DEFINE_int64(disk_reserved_bytes_free_for_testing, -1, "For testing only! Set to number of bytes free on each filesystem. " @@ -56,9 +59,6 @@ DEFINE_string(disk_reserved_override_prefix_2_path_for_testing, "", DEFINE_int64(disk_reserved_override_prefix_2_bytes_free_for_testing, -1, "For testing only! Set number of bytes free on the path prefix specified by " "--disk_reserved_override_prefix_2_path_for_testing. Set to -1 to disable."); -//DEFINE_string(disk_reserved_prefixes_with_bytes_free_for_testing, "", -// "For testing only! Syntax: '/path/a:5,/path/b:7' means a has 5 bytes free, " -// "b has 7 bytes free. Set to empty string to disable this test-specific override."); TAG_FLAG(disk_reserved_override_prefix_1_path_for_testing, unsafe); TAG_FLAG(disk_reserved_override_prefix_2_path_for_testing, unsafe); TAG_FLAG(disk_reserved_override_prefix_1_bytes_free_for_testing, unsafe); @@ -69,6 +69,7 @@ TAG_FLAG(disk_reserved_override_prefix_2_bytes_free_for_testing, runtime); using std::shared_ptr; using std::string; using std::unique_ptr; +using std::vector; using strings::Substitute; namespace kudu { @@ -189,6 +190,27 @@ Status CreateDirIfMissing(Env* env, const string& path, bool* created) { return s.IsAlreadyPresent() ? Status::OK() : s; } +Status CreateDirsRecursively(Env* env, const string& path) { + vector<string> segments = SplitPath(path); + string partial_path; + for (const string& segment : segments) { + partial_path = partial_path.empty() ? segment : JoinPathSegments(partial_path, segment); + bool is_dir; + Status s = env->IsDirectory(partial_path, &is_dir); + if (s.ok()) { + // We didn't get a NotFound error, so something is there. + if (is_dir) continue; // It's a normal directory. + // Maybe a file or a symlink. Let's try to follow the symlink. + string real_partial_path; + RETURN_NOT_OK(env->Canonicalize(partial_path, &real_partial_path)); + s = env->IsDirectory(real_partial_path, &is_dir); + if (s.ok() && is_dir) continue; // It's a symlink to a directory. + } + RETURN_NOT_OK_PREPEND(env->CreateDir(partial_path), "Unable to create directory"); + } + return Status::OK(); +} + Status CopyFile(Env* env, const string& source_path, const string& dest_path, WritableFileOptions opts) { unique_ptr<SequentialFile> source; http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/env_util.h ---------------------------------------------------------------------- diff --git a/src/kudu/util/env_util.h b/src/kudu/util/env_util.h index f3f40ae..c54bfa8 100644 --- a/src/kudu/util/env_util.h +++ b/src/kudu/util/env_util.h @@ -68,6 +68,12 @@ Status ReadFully(RandomAccessFile* file, uint64_t offset, size_t n, Status CreateDirIfMissing(Env* env, const std::string& path, bool* created = NULL); +// Recursively create directories, if they do not exist, along the given path. +// Returns OK if successful or if the given path already existed. +// Upon failure, it is possible that some part of the directory structure may +// have been successfully created. Emulates the behavior of `mkdir -p`. +Status CreateDirsRecursively(Env* env, const std::string& path); + // Copy the contents of file source_path to file dest_path. // This is not atomic, and if there is an error while reading or writing, // a partial copy may be left in 'dest_path'. Does not fsync the parent http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/path_util-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/path_util-test.cc b/src/kudu/util/path_util-test.cc index d36eba6..0d617fc 100644 --- a/src/kudu/util/path_util-test.cc +++ b/src/kudu/util/path_util-test.cc @@ -15,10 +15,16 @@ // specific language governing permissions and limitations // under the License. +#include <string> +#include <vector> + #include <gtest/gtest.h> #include "kudu/util/path_util.h" +using std::string; +using std::vector; + namespace kudu { TEST(TestPathUtil, BaseNameTest) { @@ -58,4 +64,14 @@ TEST(TestPathUtil, DirNameTest) { ASSERT_EQ("/ab", DirName("/ab/cd")); } +TEST(TestPathUtil, SplitPathTest) { + typedef vector<string> Vec; + ASSERT_EQ(Vec({"/"}), SplitPath("/")); + ASSERT_EQ(Vec({"/", "a", "b"}), SplitPath("/a/b")); + ASSERT_EQ(Vec({"/", "a", "b"}), SplitPath("/a/b/")); + ASSERT_EQ(Vec({"a", "b"}), SplitPath("a/b")); + ASSERT_EQ(Vec({"."}), SplitPath(".")); + ASSERT_EQ(Vec(), SplitPath("")); +} + } // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/path_util.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/path_util.cc b/src/kudu/util/path_util.cc index 0cbf890..6a095cc 100644 --- a/src/kudu/util/path_util.cc +++ b/src/kudu/util/path_util.cc @@ -24,12 +24,16 @@ #include <string> #include "kudu/gutil/gscoped_ptr.h" +#include "kudu/gutil/strings/split.h" #if defined(__APPLE__) #include <mutex> #endif // defined(__APPLE__) using std::string; +using std::vector; +using strings::SkipEmpty; +using strings::Split; namespace kudu { @@ -49,6 +53,17 @@ std::string JoinPathSegments(const std::string &a, } } +vector<string> SplitPath(const string& path) { + if (path.empty()) return {}; + vector<string> segments; + if (path[0] == '/') segments.push_back("/"); + vector<StringPiece> pieces = Split(path, "/", SkipEmpty()); + for (const StringPiece& piece : pieces) { + segments.emplace_back(piece.data(), piece.size()); + } + return segments; +} + string DirName(const string& path) { gscoped_ptr<char[], FreeDeleter> path_copy(strdup(path.c_str())); #if defined(__APPLE__) http://git-wip-us.apache.org/repos/asf/kudu/blob/8598bd94/src/kudu/util/path_util.h ---------------------------------------------------------------------- diff --git a/src/kudu/util/path_util.h b/src/kudu/util/path_util.h index 4b5d082..bb5c631 100644 --- a/src/kudu/util/path_util.h +++ b/src/kudu/util/path_util.h @@ -20,6 +20,7 @@ #define KUDU_UTIL_PATH_UTIL_H #include <string> +#include <vector> namespace kudu { @@ -33,6 +34,9 @@ extern const char kOldTmpInfix[]; std::string JoinPathSegments(const std::string &a, const std::string &b); +// Split a path into segments with the appropriate path separator. +std::vector<std::string> SplitPath(const std::string& path); + // Return the enclosing directory of path. // This is like dirname(3) but for C++ strings. std::string DirName(const std::string& path);
