Repository: kudu Updated Branches: refs/heads/branch-1.6.x 8d9eff378 -> 1042e4164
util: move ListFilesInDir to env_util ListFilesInDir() is useful outside of just inspecting external mini clusters. This patch moves it into env_util. Change-Id: I8372c13b28c3c3058322fc96b656ef71aa18fc0a Reviewed-on: http://gerrit.cloudera.org:8080/9166 Tested-by: Kudu Jenkins Reviewed-by: David Ribeiro Alves <[email protected]> Reviewed-on: http://gerrit.cloudera.org:8080/9437 Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/2bf90b46 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/2bf90b46 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/2bf90b46 Branch: refs/heads/branch-1.6.x Commit: 2bf90b464ed889a7e7314f764861834991cf2225 Parents: 8d9eff3 Author: Andrew Wong <[email protected]> Authored: Wed Jan 31 10:31:42 2018 -0800 Committer: Andrew Wong <[email protected]> Committed: Sat Feb 24 02:44:22 2018 +0000 ---------------------------------------------------------------------- .../external_mini_cluster_fs_inspector.cc | 26 ++++++-------------- .../external_mini_cluster_fs_inspector.h | 2 -- .../integration-tests/multidir_cluster-itest.cc | 10 ++++---- src/kudu/util/env_util.cc | 15 +++++++++++ src/kudu/util/env_util.h | 5 ++++ 5 files changed, 32 insertions(+), 26 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/2bf90b46/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc index f22f12f..15b454f 100644 --- a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc +++ b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.cc @@ -29,6 +29,7 @@ #include "kudu/fs/fs_manager.h" #include "kudu/mini-cluster/external_mini_cluster.h" #include "kudu/util/env.h" +#include "kudu/util/env_util.h" #include "kudu/util/monotime.h" #include "kudu/util/path_util.h" #include "kudu/util/pb_util.h" @@ -42,6 +43,7 @@ using std::string; using std::vector; using cluster::ExternalMiniCluster; using consensus::ConsensusMetadataPB; +using env_util::ListFilesInDir; using strings::Substitute; using tablet::TabletDataState; using tablet::TabletSuperBlockPB; @@ -53,24 +55,10 @@ ExternalMiniClusterFsInspector::ExternalMiniClusterFsInspector(ExternalMiniClust ExternalMiniClusterFsInspector::~ExternalMiniClusterFsInspector() {} -Status ExternalMiniClusterFsInspector::ListFilesInDir(const string& path, - vector<string>* entries) { - RETURN_NOT_OK(env_->GetChildren(path, entries)); - auto iter = entries->begin(); - while (iter != entries->end()) { - if (*iter == "." || *iter == ".." || iter->find(kTmpInfix) != string::npos) { - iter = entries->erase(iter); - continue; - } - ++iter; - } - return Status::OK(); -} - int ExternalMiniClusterFsInspector::CountFilesInDir(const string& path, StringPiece pattern) { vector<string> entries; - Status s = ListFilesInDir(path, &entries); + Status s = ListFilesInDir(env_, path, &entries); if (!s.ok()) return 0; return std::count_if(entries.begin(), entries.end(), [&](const string& s) { return pattern.empty() || MatchPattern(s, pattern); @@ -81,7 +69,7 @@ int ExternalMiniClusterFsInspector::CountWALFilesOnTS(int index) { string ts_wal_dir = JoinPathSegments(cluster_->tablet_server(index)->wal_dir(), FsManager::kWalDirName); vector<string> tablets; - CHECK_OK(ListFilesInDir(ts_wal_dir, &tablets)); + CHECK_OK(ListFilesInDir(env_, ts_wal_dir, &tablets)); int total_segments = 0; for (const string& tablet : tablets) { string tablet_wal_dir = JoinPathSegments(ts_wal_dir, tablet); @@ -103,7 +91,7 @@ vector<string> ExternalMiniClusterFsInspector::ListTabletsOnTS(int index) { string data_dir = cluster_->tablet_server(index)->data_dirs()[0]; string meta_dir = JoinPathSegments(data_dir, FsManager::kTabletMetadataDirName); vector<string> tablets; - CHECK_OK(ListFilesInDir(meta_dir, &tablets)); + CHECK_OK(ListFilesInDir(env_, meta_dir, &tablets)); return tablets; } @@ -111,7 +99,7 @@ vector<string> ExternalMiniClusterFsInspector::ListTabletsWithDataOnTS(int index string wal_dir = JoinPathSegments(cluster_->tablet_server(index)->wal_dir(), FsManager::kWalDirName); vector<string> tablets; - CHECK_OK(ListFilesInDir(wal_dir, &tablets)); + CHECK_OK(ListFilesInDir(env_, wal_dir, &tablets)); return tablets; } @@ -342,7 +330,7 @@ Status ExternalMiniClusterFsInspector::WaitForFilePatternInTabletWalDirOnTs( string error_msg; vector<string> entries; while (true) { - Status s = ListFilesInDir(tablet_wal_dir, &entries); + Status s = ListFilesInDir(env_, tablet_wal_dir, &entries); std::sort(entries.begin(), entries.end()); error_msg = ""; http://git-wip-us.apache.org/repos/asf/kudu/blob/2bf90b46/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h index 3fde5f9..fd82a89 100644 --- a/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h +++ b/src/kudu/integration-tests/external_mini_cluster_fs_inspector.h @@ -49,8 +49,6 @@ class ExternalMiniClusterFsInspector { explicit ExternalMiniClusterFsInspector(cluster::ExternalMiniCluster* cluster); ~ExternalMiniClusterFsInspector(); - Status ListFilesInDir(const std::string& path, std::vector<std::string>* entries); - // If provided, files are filtered by the glob-style pattern 'pattern'. int CountFilesInDir(const std::string& path, StringPiece pattern = StringPiece()); http://git-wip-us.apache.org/repos/asf/kudu/blob/2bf90b46/src/kudu/integration-tests/multidir_cluster-itest.cc ---------------------------------------------------------------------- diff --git a/src/kudu/integration-tests/multidir_cluster-itest.cc b/src/kudu/integration-tests/multidir_cluster-itest.cc index b9d6cb4..25dd253 100644 --- a/src/kudu/integration-tests/multidir_cluster-itest.cc +++ b/src/kudu/integration-tests/multidir_cluster-itest.cc @@ -26,11 +26,10 @@ #include "kudu/gutil/map-util.h" #include "kudu/integration-tests/external_mini_cluster-itest-base.h" -#include "kudu/integration-tests/external_mini_cluster_fs_inspector.h" #include "kudu/integration-tests/test_workload.h" #include "kudu/mini-cluster/external_mini_cluster.h" +#include "kudu/util/env_util.h" #include "kudu/util/path_util.h" -#include "kudu/util/status.h" #include "kudu/util/test_macros.h" #include "kudu/util/test_util.h" @@ -38,6 +37,7 @@ namespace kudu { using cluster::ExternalMiniClusterOptions; using cluster::ExternalTabletServer; +using env_util::ListFilesInDir; using std::map; using std::string; using std::vector; @@ -74,7 +74,7 @@ TEST_F(MultiDirClusterITest, TestBasicMultiDirCluster) { for (const string& data_dir : ts->data_dirs()) { string data_path = JoinPathSegments(data_dir, "data"); vector<string> files; - ASSERT_OK(inspect_->ListFilesInDir(data_path, &files)); + ASSERT_OK(ListFilesInDir(env_, data_path, &files)); InsertOrDie(&num_files_in_each_dir, data_dir, files.size()); } @@ -85,7 +85,7 @@ TEST_F(MultiDirClusterITest, TestBasicMultiDirCluster) { for (const string& data_dir : ts->data_dirs()) { string data_path = JoinPathSegments(data_dir, "data"); vector<string> files; - inspect_->ListFilesInDir(data_path, &files); + ListFilesInDir(env_, data_path, &files); int* num_files_before_insert = FindOrNull(num_files_in_each_dir, data_dir); ASSERT_NE(nullptr, num_files_before_insert); if (*num_files_before_insert < files.size()) { @@ -96,7 +96,7 @@ TEST_F(MultiDirClusterITest, TestBasicMultiDirCluster) { // data written to it. ASSERT_GT(num_dirs_added_to, 1); vector<string> wal_files; - ASSERT_OK(inspect_->ListFilesInDir(JoinPathSegments(ts->wal_dir(), "wals"), &wal_files)); + ASSERT_OK(ListFilesInDir(env_, JoinPathSegments(ts->wal_dir(), "wals"), &wal_files)); ASSERT_FALSE(wal_files.empty()); }); work.StopAndJoin(); http://git-wip-us.apache.org/repos/asf/kudu/blob/2bf90b46/src/kudu/util/env_util.cc ---------------------------------------------------------------------- diff --git a/src/kudu/util/env_util.cc b/src/kudu/util/env_util.cc index 4dbe6d0..dde4138 100644 --- a/src/kudu/util/env_util.cc +++ b/src/kudu/util/env_util.cc @@ -301,5 +301,20 @@ Status SyncAllParentDirs(Env* env, return Status::OK(); } +Status ListFilesInDir(Env* env, + const string& path, + vector<string>* entries) { + RETURN_NOT_OK(env->GetChildren(path, entries)); + auto iter = entries->begin(); + while (iter != entries->end()) { + if (*iter == "." || *iter == ".." || iter->find(kTmpInfix) != string::npos) { + iter = entries->erase(iter); + continue; + } + ++iter; + } + return Status::OK(); +} + } // namespace env_util } // namespace kudu http://git-wip-us.apache.org/repos/asf/kudu/blob/2bf90b46/src/kudu/util/env_util.h ---------------------------------------------------------------------- diff --git a/src/kudu/util/env_util.h b/src/kudu/util/env_util.h index bc630e0..de1cc3c 100644 --- a/src/kudu/util/env_util.h +++ b/src/kudu/util/env_util.h @@ -101,6 +101,11 @@ Status SyncAllParentDirs(Env* env, const std::vector<std::string>& dirs, const std::vector<std::string>& files); +// Return a list of files within the given 'path'. +Status ListFilesInDir(Env* env, + const std::string& path, + std::vector<std::string>* entries); + } // namespace env_util } // namespace kudu
