Repository: kudu Updated Branches: refs/heads/master 9199d1646 -> 368990140
KUDU-2531: (part 1) Ignore invalid tablet metadata files Changes the behavior of FsManager::IsValidTabletId to return false for all files that are named with invalid tabet ids. This results in all files that are not valid table ids being ignored, instead of just tmp files. Change-Id: If9f0e6dd8925519aa3ed3e735f2987743f2ef1d1 Reviewed-on: http://gerrit.cloudera.org:8080/11259 Tested-by: Kudu Jenkins Reviewed-by: Grant Henke <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/kudu/repo Commit: http://git-wip-us.apache.org/repos/asf/kudu/commit/6d7d8f55 Tree: http://git-wip-us.apache.org/repos/asf/kudu/tree/6d7d8f55 Diff: http://git-wip-us.apache.org/repos/asf/kudu/diff/6d7d8f55 Branch: refs/heads/master Commit: 6d7d8f555e85bd2809fe9dea0630e2820ac2a892 Parents: 9199d16 Author: Grant Henke <[email protected]> Authored: Fri Aug 17 13:54:03 2018 -0500 Committer: Grant Henke <[email protected]> Committed: Tue Aug 21 03:31:53 2018 +0000 ---------------------------------------------------------------------- src/kudu/fs/fs_manager-test.cc | 10 +++++++++- src/kudu/fs/fs_manager.cc | 33 ++++++++++++++++++++------------- src/kudu/fs/fs_manager.h | 8 +++++++- 3 files changed, 36 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kudu/blob/6d7d8f55/src/kudu/fs/fs_manager-test.cc ---------------------------------------------------------------------- diff --git a/src/kudu/fs/fs_manager-test.cc b/src/kudu/fs/fs_manager-test.cc index 194ecef..df84259 100644 --- a/src/kudu/fs/fs_manager-test.cc +++ b/src/kudu/fs/fs_manager-test.cc @@ -181,9 +181,17 @@ TEST_F(FsManagerTestBase, TestListTablets) { ASSERT_OK(env_->NewWritableFile( JoinPathSegments(path, "foo.kudutmp.abc123"), &writer)); ASSERT_OK(env_->NewWritableFile( + JoinPathSegments(path, "foo.bak"), &writer)); + ASSERT_OK(env_->NewWritableFile( + JoinPathSegments(path, "foo.bak.abc123"), &writer)); + ASSERT_OK(env_->NewWritableFile( JoinPathSegments(path, ".hidden"), &writer)); + // An uncanonicalized id. + ASSERT_OK(env_->NewWritableFile( + JoinPathSegments(path, "6ba7b810-9dad-11d1-80b4-00c04fd430c8"), &writer)); + // 1 valid tablet id. ASSERT_OK(env_->NewWritableFile( - JoinPathSegments(path, "a_tablet_sort_of"), &writer)); + JoinPathSegments(path, "922ff7ed14c14dbca4ee16331dfda42a"), &writer)); ASSERT_OK(fs_manager()->ListTabletIds(&tablet_ids)); ASSERT_EQ(1, tablet_ids.size()) << tablet_ids; http://git-wip-us.apache.org/repos/asf/kudu/blob/6d7d8f55/src/kudu/fs/fs_manager.cc ---------------------------------------------------------------------- diff --git a/src/kudu/fs/fs_manager.cc b/src/kudu/fs/fs_manager.cc index 8c08f34..93fd7f3 100644 --- a/src/kudu/fs/fs_manager.cc +++ b/src/kudu/fs/fs_manager.cc @@ -57,6 +57,7 @@ #include "kudu/util/path_util.h" #include "kudu/util/pb_util.h" #include "kudu/util/scoped_cleanup.h" +#include "kudu/util/slice.h" #include "kudu/util/stopwatch.h" DEFINE_bool(enable_data_block_fsync, true, @@ -569,13 +570,12 @@ Status FsManager::CreateFileSystemRoots( Status FsManager::CreateInstanceMetadata(boost::optional<string> uuid, InstanceMetadataPB* metadata) { - ObjectIdGenerator oid_generator; if (uuid) { string canonicalized_uuid; - RETURN_NOT_OK(oid_generator.Canonicalize(uuid.get(), &canonicalized_uuid)); + RETURN_NOT_OK(oid_generator_.Canonicalize(uuid.get(), &canonicalized_uuid)); metadata->set_uuid(canonicalized_uuid); } else { - metadata->set_uuid(oid_generator.Next()); + metadata->set_uuid(oid_generator_.Next()); } string time_str; @@ -621,24 +621,31 @@ string FsManager::GetTabletMetadataPath(const string& tablet_id) const { return JoinPathSegments(GetTabletMetadataDir(), tablet_id); } -namespace { -// Return true if 'fname' is a valid tablet ID. -bool IsValidTabletId(const string& fname) { - if (fname.find(kTmpInfix) != string::npos || - fname.find(kOldTmpInfix) != string::npos) { - LOG(WARNING) << "Ignoring tmp file in tablet metadata dir: " << fname; +bool FsManager::IsValidTabletId(const string& fname) { + // Prevent warning logs for hidden files or ./.. + if (HasPrefixString(fname, ".")) { + VLOG(1) << "Ignoring hidden file in tablet metadata dir: " << fname; return false; } - if (HasPrefixString(fname, ".")) { - // Hidden file or ./.. - VLOG(1) << "Ignoring hidden file in tablet metadata dir: " << fname; + string canonicalized_uuid; + Status s = oid_generator_.Canonicalize(fname, &canonicalized_uuid); + + if (!s.ok()) { + LOG(WARNING) << "Ignoring file in tablet metadata dir: " << fname << ": " << + s.message().ToString(); + return false; + } + + if (fname != canonicalized_uuid) { + LOG(WARNING) << "Ignoring file in tablet metadata dir: " << fname << ": " << + Substitute("canonicalized uuid $0 does not match file name", + canonicalized_uuid); return false; } return true; } -} // anonymous namespace Status FsManager::ListTabletIds(vector<string>* tablet_ids) { string dir = GetTabletMetadataDir(); http://git-wip-us.apache.org/repos/asf/kudu/blob/6d7d8f55/src/kudu/fs/fs_manager.h ---------------------------------------------------------------------- diff --git a/src/kudu/fs/fs_manager.h b/src/kudu/fs/fs_manager.h index 9a29e57..c71cb99 100644 --- a/src/kudu/fs/fs_manager.h +++ b/src/kudu/fs/fs_manager.h @@ -34,6 +34,7 @@ #include "kudu/gutil/ref_counted.h" #include "kudu/util/env.h" #include "kudu/util/metrics.h" +#include "kudu/util/oid_generator.h" #include "kudu/util/path_util.h" #include "kudu/util/status.h" @@ -50,9 +51,9 @@ namespace fs { class BlockManager; class ReadableBlock; class WritableBlock; +struct CreateBlockOptions; struct FsReport; -struct CreateBlockOptions; } // namespace fs namespace itest { @@ -314,6 +315,9 @@ class FsManager { // configured umask, and tightens them as necessary if they do not. void CheckAndFixPermissions(); + // Returns true if 'fname' is a valid tablet ID. + bool IsValidTabletId(const std::string& fname); + static const char *kDataDirName; static const char *kTabletMetadataDirName; static const char *kWalDirName; @@ -345,6 +349,8 @@ class FsManager { std::unique_ptr<fs::DataDirManager> dd_manager_; std::unique_ptr<fs::BlockManager> block_manager_; + ObjectIdGenerator oid_generator_; + bool initted_; DISALLOW_COPY_AND_ASSIGN(FsManager);
