This is an automated email from the ASF dual-hosted git repository. laiyingchun pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit 9181f9ebef293229307dae0d84461e8616747c57 Author: xinghuayu007 <[email protected]> AuthorDate: Wed Oct 18 16:46:41 2023 +0800 [Tool] Find file path where the block is located Sometimes, a block is damaged because of the checksum is incorrect or the disk is damaged. The log only prints the block id, but no container id. We want to know the container where block is located. If we know the container id, we can read the related metadata file to get the status of the block: dead or alive. We can also truncate the record from the metadata file to repair the container, although it will cause some data loss. This patch provides the tool to find the file path by a block id. Change-Id: Ifaafd2bb634ed7fab923d9cf9eef40437442d187 Reviewed-on: http://gerrit.cloudera.org:8080/20594 Tested-by: Kudu Jenkins Reviewed-by: Yingchun Lai <[email protected]> --- src/kudu/fs/block_manager.h | 7 +++++ src/kudu/fs/file_block_manager.h | 7 ++--- src/kudu/fs/log_block_manager.cc | 15 ++++++++++ src/kudu/fs/log_block_manager.h | 2 ++ src/kudu/tools/kudu-tool-test.cc | 64 ++++++++++++++++++++++++++++++++++++++++ src/kudu/tools/tool_action_fs.cc | 45 ++++++++++++++++++++++++++++ 6 files changed, 135 insertions(+), 5 deletions(-) diff --git a/src/kudu/fs/block_manager.h b/src/kudu/fs/block_manager.h index 382f68469..264d22507 100644 --- a/src/kudu/fs/block_manager.h +++ b/src/kudu/fs/block_manager.h @@ -258,6 +258,13 @@ class BlockManager : public RefCountedThreadSafe<BlockManager> { virtual Status OpenBlock(const BlockId& block_id, std::unique_ptr<ReadableBlock>* block) = 0; + // Get the data file where the block located. + // + // Find the data file according to the block id. The data file is the + // unique file where the block data is stored. On success, overwrites + // 'path' with the file's path. + virtual bool FindBlockPath(const BlockId& block_id, std::string* path) const = 0; + // Constructs a block creation transaction to group a set of block creation // operations and closes the registered blocks together. virtual std::unique_ptr<BlockCreationTransaction> NewCreationTransaction() = 0; diff --git a/src/kudu/fs/file_block_manager.h b/src/kudu/fs/file_block_manager.h index d770e2386..81ab17561 100644 --- a/src/kudu/fs/file_block_manager.h +++ b/src/kudu/fs/file_block_manager.h @@ -89,6 +89,8 @@ class FileBlockManager : public BlockManager { Status CreateBlock(const CreateBlockOptions& opts, std::unique_ptr<WritableBlock>* block) override; + bool FindBlockPath(const BlockId& block_id, std::string* path) const override; + Status OpenBlock(const BlockId& block_id, std::unique_ptr<ReadableBlock>* block) override; @@ -121,11 +123,6 @@ class FileBlockManager : public BlockManager { // Synchronizes the metadata for a block with the given location. Status SyncMetadata(const internal::FileBlockLocation& location); - // Looks up the path of the file backing a particular block ID. - // - // On success, overwrites 'path' with the file's path. - bool FindBlockPath(const BlockId& block_id, std::string* path) const; - Env* env() const { return env_; } // For manipulating files. diff --git a/src/kudu/fs/log_block_manager.cc b/src/kudu/fs/log_block_manager.cc index 610758317..d8b43e742 100644 --- a/src/kudu/fs/log_block_manager.cc +++ b/src/kudu/fs/log_block_manager.cc @@ -2641,6 +2641,21 @@ Status LogBlockManager::OpenBlock(const BlockId& block_id, return Status::OK(); } +bool LogBlockManager::FindBlockPath(const BlockId& block_id, string* path) const { + LogBlockRefPtr lb; + { + auto index = block_id.id() & kBlockMapMask; + std::lock_guard<simple_spinlock> l(*managed_block_shards_[index].lock); + lb = FindPtrOrNull(*managed_block_shards_[index].blocks_by_block_id, block_id); + } + if (!lb) { + return false; + } + *path = StrCat(JoinPathSegments(lb->container()->data_dir()->dir(), lb->container()->id()), + LogBlockManager::kContainerDataFileSuffix); + return true; +} + unique_ptr<BlockCreationTransaction> LogBlockManager::NewCreationTransaction() { CHECK(!opts_.read_only); return std::make_unique<internal::LogBlockCreationTransaction>(); diff --git a/src/kudu/fs/log_block_manager.h b/src/kudu/fs/log_block_manager.h index 0cc8d4d25..f2d20bb57 100644 --- a/src/kudu/fs/log_block_manager.h +++ b/src/kudu/fs/log_block_manager.h @@ -191,6 +191,8 @@ class LogBlockManager : public BlockManager { Status OpenBlock(const BlockId& block_id, std::unique_ptr<ReadableBlock>* block) override; + bool FindBlockPath(const BlockId& block_id, std::string* path) const override; + std::unique_ptr<BlockCreationTransaction> NewCreationTransaction() override; std::shared_ptr<BlockDeletionTransaction> NewDeletionTransaction() override; diff --git a/src/kudu/tools/kudu-tool-test.cc b/src/kudu/tools/kudu-tool-test.cc index 803c8bf17..ec311ea12 100644 --- a/src/kudu/tools/kudu-tool-test.cc +++ b/src/kudu/tools/kudu-tool-test.cc @@ -1366,6 +1366,7 @@ TEST_F(ToolTest, TestModeHelp) { "dump.*Dump a Kudu filesystem", "format.*new Kudu filesystem", "list.*List metadata for on-disk tablets, rowsets", + "locate_block.*Find the file's path where the block locates", "update_dirs.*Updates the set of data directories", "upgrade_encryption_key.*Upgrade the encryption key info", }; @@ -9303,6 +9304,69 @@ TEST_F(UnregisterTServerTest, TestUnregisterTServerNotPresumedDead) { } } +class FindBlockPathTest : public ToolTest, public ::testing::WithParamInterface<string> { +}; + +static vector<string> BlockManagerType() { + return { "log", "file" }; +} + +INSTANTIATE_TEST_SUITE_P(, FindBlockPathTest, ::testing::ValuesIn(BlockManagerType())); + +TEST_P(FindBlockPathTest, TestFindBlockPath) { + FLAGS_block_manager = GetParam(); + + // Create replicas and fill some data. + NO_FATALS(StartMiniCluster()); + NO_FATALS(CreateTableWithFlushedData("tablename", mini_cluster_.get())); + + auto fs_manager = mini_cluster_->mini_tablet_server(0)->server()->fs_manager(); + vector<BlockId> block_ids; + ASSERT_OK(fs_manager->block_manager()->GetAllBlockIds(&block_ids)); + string fs_wal_dir = mini_cluster_->mini_tablet_server(0)->options()->fs_opts.wal_root; + string fs_data_dirs = JoinMapped(fs_manager->GetDataRootDirs(), + [] (const string& data_dir) { + return data_dir.substr(0, data_dir.length() - strlen("/data")); + }, ","); + string fs_paths = "--fs_wal_dir=" + fs_wal_dir + " " + "--fs_data_dirs=" + fs_data_dirs; + + ASSERT_EQ(1, fs_manager->GetDataRootDirs().size()); + + // Case 1: block id exists. + string stdout; + NO_FATALS(RunActionStdoutString( + Substitute("fs locate_block $0 $1 --block_manager=$2", + block_ids[0].ToString(), fs_paths, FLAGS_block_manager), + &stdout)); + ASSERT_STR_CONTAINS(stdout, Substitute("$0 | $1", + block_ids[0].ToString(), fs_data_dirs)); + + // Case 2: block id does not exists. + stdout.clear(); + // 281474976710655 = (2^49 - 1). That is a very unique id, which will + // not exist in log_blog_manager or file_block_manager. + NO_FATALS(RunActionStdoutString( + Substitute("fs locate_block 281474976710655,$0 $1 --block_manager=$2", + block_ids[0].ToString(), fs_paths, FLAGS_block_manager), + &stdout)); + ASSERT_STR_CONTAINS(stdout, Substitute("$0 | $1", + block_ids[0].ToString(), fs_data_dirs)); + ASSERT_STR_CONTAINS(stdout, Substitute("$0 | $1", + block_ids[0].ToString(), fs_data_dirs)); + ASSERT_STR_MATCHES(stdout, "281474976710655* | not found"); + + // case 3: block id is not integer. + string stderr; + NO_FATALS(RunActionStdoutString( + Substitute("fs locate_block 88uu88888,$0 $1 --block_manager=$2", + block_ids[0].ToString(), fs_paths, FLAGS_block_manager), + &stdout)); + ASSERT_STR_CONTAINS(stdout, Substitute("$0 | $1", + block_ids[0].ToString(), fs_data_dirs)); + ASSERT_STR_MATCHES(stdout, "88uu88888* | not an integer"); +} + TEST_F(ToolTest, TestLocalReplicaCopyLocal) { SKIP_IF_SLOW_NOT_ALLOWED(); // TODO(abukor): Rewrite the test to make sure it works with encryption diff --git a/src/kudu/tools/tool_action_fs.cc b/src/kudu/tools/tool_action_fs.cc index 848064ffc..8085d821c 100644 --- a/src/kudu/tools/tool_action_fs.cc +++ b/src/kudu/tools/tool_action_fs.cc @@ -25,6 +25,7 @@ #include <string> #include <type_traits> #include <unordered_map> +#include <unordered_set> #include <utility> #include <vector> @@ -168,9 +169,13 @@ using std::shared_ptr; using std::string; using std::unique_ptr; using std::unordered_map; +using std::unordered_set; using std::vector; using strings::Substitute; +constexpr const char* const kBlockIdsArg = "block_ids"; +constexpr const char* const kBlockIdsDesc = "Block ids existing in metadata files"; + namespace kudu { namespace tools { @@ -439,6 +444,36 @@ Status DumpFsTree(const RunnerContext& /*context*/) { return Status::OK(); } +Status LocateBlock(const RunnerContext& context) { + const string& block_ids_str = FindOrDie(context.required_args, kBlockIdsArg); + vector<string> block_ids = strings::Split(block_ids_str, ",", strings::SkipEmpty()); + if (block_ids.empty()) { + return Status::InvalidArgument("no block identifiers provided"); + } + FsManagerOpts opts; + opts.read_only = true; + opts.update_instances = UpdateInstanceBehavior::DONT_UPDATE; + FsManager fs_manager(Env::Default(), std::move(opts)); + RETURN_NOT_OK(fs_manager.Open()); + DataTable output_table({ "block id", "path" }); + for (const string& blkid_str : block_ids) { + uint64_t blkid = 0; + if (!safe_strtou64(blkid_str.c_str(), &blkid)) { + output_table.AddRow({blkid_str, "not an integer"}); + continue; + } + BlockId block_id(blkid); + string path; + if (fs_manager.block_manager()->FindBlockPath(block_id, &path)) { + output_table.AddRow({blkid_str, path}); + } else { + output_table.AddRow({blkid_str, "not found"}); + } + } + RETURN_NOT_OK(output_table.PrintTo(cout)); + return Status::OK(); +} + Status CheckForTabletsThatWillFailWithUpdate() { FsManagerOpts opts; opts.read_only = true; @@ -1068,12 +1103,22 @@ unique_ptr<Mode> BuildFsMode() { .AddOptionalParameter("fs_wal_dir") .Build(); + unique_ptr<Action> locate_block = + ActionBuilder("locate_block", &LocateBlock) + .Description("Find the file's path where the block locates") + .AddRequiredParameter({ kBlockIdsArg, kBlockIdsDesc }) + .AddOptionalParameter("fs_data_dirs") + .AddOptionalParameter("fs_metadata_dir") + .AddOptionalParameter("fs_wal_dir") + .Build(); + return ModeBuilder("fs") .Description("Operate on a local Kudu filesystem") .AddMode(BuildFsDumpMode()) .AddAction(std::move(check)) .AddAction(std::move(format)) .AddAction(std::move(list)) + .AddAction(std::move(locate_block)) .AddAction(std::move(update)) .AddAction(std::move(upgrade_encryption_key)) .Build();
