This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 9bb9bc08cd6 branch-3.1: [tool](filecache) add debugging facilities for
lru dump #54412 (#58871)
9bb9bc08cd6 is described below
commit 9bb9bc08cd69a28848923bdac79a0f79be36923f
Author: zhengyu <[email protected]>
AuthorDate: Wed Dec 10 12:15:04 2025 +0800
branch-3.1: [tool](filecache) add debugging facilities for lru dump #54412
(#58871)
picked from #54412
Signed-off-by: zhengyu <[email protected]>
Signed-off-by: freemandealer <[email protected]>
---
be/CMakeLists.txt | 3 +
be/src/http/action/file_cache_action.cpp | 3 +
be/src/io/CMakeLists.txt | 25 +++
be/src/io/cache/block_file_cache.cpp | 21 +-
be/src/io/cache/block_file_cache.h | 4 +-
be/src/io/cache/block_file_cache_factory.cpp | 6 +
be/src/io/cache/block_file_cache_factory.h | 5 +
be/src/io/cache/cache_lru_dumper.cpp | 53 ++++-
be/src/io/cache/cache_lru_dumper.h | 17 +-
be/src/io/cache/cached_remote_file_reader.cpp | 7 +
be/src/io/cache/file_cache_lru_tool.cpp | 233 +++++++++++++++++++++
build.sh | 6 +
.../suites/demo_p0/test_lru_persist.groovy | 37 +++-
13 files changed, 403 insertions(+), 17 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index de46bcfeb8b..0c06094d227 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -142,6 +142,9 @@ option(WITH_MYSQL "Support access MySQL" ON)
option(BUILD_FS_BENCHMARK "ON for building fs benchmark tool or OFF for not"
OFF)
message(STATUS "build fs benchmark tool: ${BUILD_FS_BENCHMARK}")
+option(BUILD_FILE_CACHE_LRU_TOOL "ON for building file cache lru tool or OFF
for not" OFF)
+message(STATUS "build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}")
+
set(CMAKE_SKIP_RPATH TRUE)
set(Boost_USE_STATIC_LIBS ON)
set(Boost_USE_STATIC_RUNTIME ON)
diff --git a/be/src/http/action/file_cache_action.cpp
b/be/src/http/action/file_cache_action.cpp
index 4b9310435a2..882dc895480 100644
--- a/be/src/http/action/file_cache_action.cpp
+++ b/be/src/http/action/file_cache_action.cpp
@@ -54,6 +54,7 @@ constexpr static std::string_view CAPACITY = "capacity";
constexpr static std::string_view RELEASE = "release";
constexpr static std::string_view BASE_PATH = "base_path";
constexpr static std::string_view RELEASED_ELEMENTS = "released_elements";
+constexpr static std::string_view DUMP = "dump";
constexpr static std::string_view VALUE = "value";
Status FileCacheAction::_handle_header(HttpRequest* req, std::string*
json_metrics) {
@@ -132,6 +133,8 @@ Status FileCacheAction::_handle_header(HttpRequest* req,
std::string* json_metri
*json_metrics = json.ToString();
}
}
+ } else if (operation == DUMP) {
+ io::FileCacheFactory::instance()->dump_all_caches();
} else {
st = Status::InternalError("invalid operation: {}", operation);
}
diff --git a/be/src/io/CMakeLists.txt b/be/src/io/CMakeLists.txt
index dbd52e3560d..b71ff8ad0f3 100644
--- a/be/src/io/CMakeLists.txt
+++ b/be/src/io/CMakeLists.txt
@@ -63,3 +63,28 @@ if (${BUILD_FS_BENCHMARK} STREQUAL "ON")
)
endif()
+
+if (${BUILD_FILE_CACHE_LRU_TOOL} STREQUAL "ON")
+ add_executable(file_cache_lru_tool
+ cache/file_cache_lru_tool.cpp
+ )
+
+ pch_reuse(file_cache_lru_tool)
+
+ # This permits libraries loaded by dlopen to link to the symbols in the
program.
+ set_target_properties(file_cache_lru_tool PROPERTIES ENABLE_EXPORTS 1)
+
+ target_link_libraries(file_cache_lru_tool
+ ${DORIS_LINK_LIBS}
+ )
+
+ install(DIRECTORY DESTINATION ${OUTPUT_DIR}/lib/)
+ install(TARGETS file_cache_lru_tool DESTINATION ${OUTPUT_DIR}/lib/)
+
+ add_custom_command(TARGET file_cache_lru_tool POST_BUILD
+ COMMAND ${CMAKE_OBJCOPY} --only-keep-debug
$<TARGET_FILE:file_cache_lru_tool> $<TARGET_FILE:file_cache_lru_tool>.dbg
+ COMMAND ${CMAKE_STRIP} --strip-debug --strip-unneeded
$<TARGET_FILE:file_cache_lru_tool>
+ COMMAND ${CMAKE_OBJCOPY}
--add-gnu-debuglink=$<TARGET_FILE:file_cache_lru_tool>.dbg
$<TARGET_FILE:file_cache_lru_tool>
+ )
+
+endif()
diff --git a/be/src/io/cache/block_file_cache.cpp
b/be/src/io/cache/block_file_cache.cpp
index 6b2741368c4..9e1c275060a 100644
--- a/be/src/io/cache/block_file_cache.cpp
+++ b/be/src/io/cache/block_file_cache.cpp
@@ -2379,6 +2379,18 @@ void BlockFileCache::run_background_lru_log_replay() {
}
}
+void BlockFileCache::dump_lru_queues(bool force) {
+ std::unique_lock dump_lock(_dump_lru_queues_mtx);
+ if (config::file_cache_background_lru_dump_tail_record_num > 0 &&
+ !ExecEnv::GetInstance()->get_is_upgrading()) {
+ _lru_dumper->dump_queue("disposable", force);
+ _lru_dumper->dump_queue("normal", force);
+ _lru_dumper->dump_queue("index", force);
+ _lru_dumper->dump_queue("ttl", force);
+ _lru_dumper->set_first_dump_done();
+ }
+}
+
void BlockFileCache::run_background_lru_dump() {
Thread::set_self_name("run_background_lru_dump");
while (!_close) {
@@ -2390,14 +2402,7 @@ void BlockFileCache::run_background_lru_dump() {
break;
}
}
-
- if (config::file_cache_background_lru_dump_tail_record_num > 0 &&
- !ExecEnv::GetInstance()->get_is_upgrading()) {
- _lru_dumper->dump_queue("disposable");
- _lru_dumper->dump_queue("normal");
- _lru_dumper->dump_queue("index");
- _lru_dumper->dump_queue("ttl");
- }
+ dump_lru_queues(false);
}
}
diff --git a/be/src/io/cache/block_file_cache.h
b/be/src/io/cache/block_file_cache.h
index b200ec84f48..512de17b52e 100644
--- a/be/src/io/cache/block_file_cache.h
+++ b/be/src/io/cache/block_file_cache.h
@@ -207,6 +207,8 @@ public:
std::string dump_structure(const UInt128Wrapper& hash);
std::string dump_single_cache_type(const UInt128Wrapper& hash, size_t
offset);
+ void dump_lru_queues(bool force);
+
[[nodiscard]] size_t get_used_cache_size(FileCacheType type) const;
[[nodiscard]] size_t get_file_blocks_num(FileCacheType type) const;
@@ -565,7 +567,7 @@ private:
// so join this async load thread first
std::unique_ptr<FileCacheStorage> _storage;
std::shared_ptr<bvar::LatencyRecorder> _lru_dump_latency_us;
-
+ std::mutex _dump_lru_queues_mtx;
moodycamel::ConcurrentQueue<FileBlockSPtr> _need_update_lru_blocks;
};
diff --git a/be/src/io/cache/block_file_cache_factory.cpp
b/be/src/io/cache/block_file_cache_factory.cpp
index 2d7f4eb49fd..a83a81749e2 100644
--- a/be/src/io/cache/block_file_cache_factory.cpp
+++ b/be/src/io/cache/block_file_cache_factory.cpp
@@ -220,6 +220,12 @@ std::string FileCacheFactory::clear_file_caches(bool sync)
{
return ss.str();
}
+void FileCacheFactory::dump_all_caches() {
+ for (const auto& cache : _caches) {
+ cache->dump_lru_queues(true);
+ }
+}
+
std::vector<std::string> FileCacheFactory::get_base_paths() {
std::vector<std::string> paths;
for (const auto& pair : _path_to_cache) {
diff --git a/be/src/io/cache/block_file_cache_factory.h
b/be/src/io/cache/block_file_cache_factory.h
index 30d69dc15ca..837feac7f68 100644
--- a/be/src/io/cache/block_file_cache_factory.h
+++ b/be/src/io/cache/block_file_cache_factory.h
@@ -84,6 +84,11 @@ public:
*/
std::string clear_file_caches(bool sync);
+ /**
+ * dump lru queue info for all file cache instances
+ */
+ void dump_all_caches();
+
std::vector<std::string> get_base_paths();
/**
diff --git a/be/src/io/cache/cache_lru_dumper.cpp
b/be/src/io/cache/cache_lru_dumper.cpp
index cb8e6a2059d..0c0375883da 100644
--- a/be/src/io/cache/cache_lru_dumper.cpp
+++ b/be/src/io/cache/cache_lru_dumper.cpp
@@ -230,10 +230,55 @@ Status CacheLRUDumper::finalize_dump(std::ofstream& out,
size_t entry_num,
out.close();
+ if (_is_first_dump) [[unlikely]] {
+ // we back up two dumps (one for last before be restart, one for first
after be restart)
+ // for later debug the restore process
+ try {
+ if (std::filesystem::exists(final_filename)) {
+ std::string backup_filename = final_filename + "_" +
_start_time + "_last";
+ std::rename(final_filename.c_str(), backup_filename.c_str());
+ }
+ std::string timestamped_filename = final_filename + "_" +
_start_time;
+ std::filesystem::copy_file(tmp_filename, timestamped_filename);
+
+ std::filesystem::path dir =
std::filesystem::path(final_filename).parent_path();
+ std::string prefix =
std::filesystem::path(final_filename).filename().string();
+ uint64_t total_size = 0;
+ std::vector<std::pair<std::filesystem::path,
std::filesystem::file_time_type>> files;
+ for (const auto& entry : std::filesystem::directory_iterator(dir))
{
+ if (entry.path().filename().string().find(prefix) == 0) {
+ total_size += entry.file_size();
+ files.emplace_back(entry.path(), entry.last_write_time());
+ }
+ }
+ if (total_size > 5ULL * 1024 * 1024 * 1024) {
+ // delete oldest two files
+ std::sort(files.begin(), files.end(),
+ [](const auto& a, const auto& b) { return a.second <
b.second; });
+ if (!files.empty()) {
+ auto remove_file = [](const std::filesystem::path&
file_path) {
+ std::error_code ec;
+ bool removed = std::filesystem::remove(file_path, ec);
+ LOG(INFO) << "Remove " << (removed ? "succeeded" :
"failed")
+ << " for file: " << file_path
+ << (ec ? ", error: " + ec.message() : "");
+ return removed;
+ };
+
+ remove_file(files[0].first);
+ if (files.size() > 1) {
+ remove_file(files[1].first);
+ }
+ }
+ }
+ } catch (const std::filesystem::filesystem_error& e) {
+ LOG(WARNING) << "failed to handle first dump case: " << e.what();
+ }
+ }
+
// Rename tmp to formal file
try {
std::rename(tmp_filename.c_str(), final_filename.c_str());
- std::remove(tmp_filename.c_str());
file_size = std::filesystem::file_size(final_filename);
} catch (const std::filesystem::filesystem_error& e) {
LOG(WARNING) << "failed to rename " << tmp_filename << " to " <<
final_filename
@@ -247,10 +292,10 @@ Status CacheLRUDumper::finalize_dump(std::ofstream& out,
size_t entry_num,
return Status::OK();
}
-void CacheLRUDumper::dump_queue(const std::string& queue_name) {
+void CacheLRUDumper::dump_queue(const std::string& queue_name, bool force) {
FileCacheType type = string_to_cache_type(queue_name);
- if (_recorder->get_lru_queue_update_cnt_from_last_dump(type) >
- config::file_cache_background_lru_dump_update_cnt_threshold) {
+ if (force || _recorder->get_lru_queue_update_cnt_from_last_dump(type) >
+
config::file_cache_background_lru_dump_update_cnt_threshold) {
LRUQueue& queue = _recorder->get_shadow_queue(type);
do_dump_queue(queue, queue_name);
_recorder->reset_lru_queue_update_cnt_from_last_dump(type);
diff --git a/be/src/io/cache/cache_lru_dumper.h
b/be/src/io/cache/cache_lru_dumper.h
index 801ed577de2..d9addff614c 100644
--- a/be/src/io/cache/cache_lru_dumper.h
+++ b/be/src/io/cache/cache_lru_dumper.h
@@ -17,8 +17,10 @@
#pragma once
+#include <chrono>
#include <cstdint>
#include <cstring>
+#include <ctime>
#include <filesystem>
#include <fstream>
#include <iostream>
@@ -38,11 +40,19 @@ class LRUQueueRecorder;
class CacheLRUDumper {
public:
CacheLRUDumper(BlockFileCache* mgr, LRUQueueRecorder* recorder)
- : _mgr(mgr), _recorder(recorder) {};
- void dump_queue(const std::string& queue_name);
+ : _mgr(mgr), _recorder(recorder) {
+ auto now = std::chrono::system_clock::now();
+ auto in_time_t = std::chrono::system_clock::to_time_t(now);
+ std::stringstream ss;
+ ss << std::put_time(std::localtime(&in_time_t), "%Y%m%d%H%M%S");
+ _start_time = ss.str();
+ };
+
+ void dump_queue(const std::string& queue_name, bool force);
void restore_queue(LRUQueue& queue, const std::string& queue_name,
std::lock_guard<std::mutex>& cache_lock);
void remove_lru_dump_files();
+ void set_first_dump_done() { _is_first_dump = false; }
private:
void do_dump_queue(LRUQueue& queue, const std::string& queue_name);
@@ -79,5 +89,8 @@ private:
BlockFileCache* _mgr;
LRUQueueRecorder* _recorder;
+
+ std::string _start_time;
+ bool _is_first_dump = true;
};
} // namespace doris::io
\ No newline at end of file
diff --git a/be/src/io/cache/cached_remote_file_reader.cpp
b/be/src/io/cache/cached_remote_file_reader.cpp
index 9bde202a3d6..37fefe3d79a 100644
--- a/be/src/io/cache/cached_remote_file_reader.cpp
+++ b/be/src/io/cache/cached_remote_file_reader.cpp
@@ -389,6 +389,9 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset,
Slice result, size_t*
for (auto& block : holder.file_blocks) {
switch (block->state()) {
case FileBlock::State::EMPTY:
+ VLOG_DEBUG << fmt::format("Block EMPTY path={} hash={}:{}:{}
offset={} cache_path={}",
+ path().native(),
_cache_hash.to_string(), _cache_hash.high(),
+ _cache_hash.low(), block->offset(),
block->get_cache_file());
block->get_or_set_downloader();
if (block->is_downloader()) {
empty_blocks.push_back(block);
@@ -397,6 +400,10 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset,
Slice result, size_t*
stats.hit_cache = false;
break;
case FileBlock::State::SKIP_CACHE:
+ VLOG_DEBUG << fmt::format(
+ "Block SKIP_CACHE path={} hash={}:{}:{} offset={}
cache_path={}",
+ path().native(), _cache_hash.to_string(),
_cache_hash.high(), _cache_hash.low(),
+ block->offset(), block->get_cache_file());
empty_blocks.push_back(block);
stats.hit_cache = false;
stats.skip_cache = true;
diff --git a/be/src/io/cache/file_cache_lru_tool.cpp
b/be/src/io/cache/file_cache_lru_tool.cpp
new file mode 100644
index 00000000000..a6e133c7a56
--- /dev/null
+++ b/be/src/io/cache/file_cache_lru_tool.cpp
@@ -0,0 +1,233 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <gflags/gflags.h>
+
+#include <fstream>
+#include <iostream>
+#include <sstream>
+
+#include "common/status.h"
+#include "gen_cpp/file_cache.pb.h"
+#include "io/cache/cache_lru_dumper.h"
+#include "io/cache/file_cache_common.h"
+#include "io/cache/lru_queue_recorder.h"
+#include "util/coding.h"
+#include "util/crc32c.h"
+
+using namespace doris;
+
+DEFINE_string(filename, "", "dump file name");
+
+std::string get_usage(const std::string& progname) {
+ std::stringstream ss;
+ ss << progname << " is the Doris BE file cache lru tool for examing dumped
content.\n";
+
+ ss << "Usage:\n";
+ ss << progname << " --filename [filename]\n";
+ ss << "\nExample:\n";
+ ss << progname << " --filename ./lru_dump_ttl.tail\n";
+ return ss.str();
+}
+
+Status check_ifstream_status(std::ifstream& in, std::string& filename) {
+ if (!in.good()) {
+ std::ios::iostate state = in.rdstate();
+ std::stringstream err_msg;
+ if (state & std::ios::eofbit) {
+ err_msg << "End of file reached.";
+ }
+ if (state & std::ios::failbit) {
+ err_msg << "Input/output operation failed, err_code: " <<
strerror(errno);
+ }
+ if (state & std::ios::badbit) {
+ err_msg << "Serious I/O error occurred, err_code: " <<
strerror(errno);
+ }
+ in.close();
+ std::string warn_msg = std::string(
+ fmt::format("dump lru reading failed, file={}, {}", filename,
err_msg.str()));
+ std::cerr << warn_msg << std::endl;
+ return Status::InternalError<false>(warn_msg);
+ }
+
+ return Status::OK();
+}
+
+struct Footer {
+ size_t meta_offset;
+ uint32_t checksum;
+ uint8_t version;
+ char magic[3];
+
+ std::string serialize_as_string() const;
+ bool deserialize_from_string(const std::string& data) {
+ DCHECK(data.size() == sizeof(Footer));
+
+ const char* ptr = data.data();
+
+ // Deserialize meta_offset (convert from little-endian)
+ uint64_t meta_offset_le;
+ std::memcpy(&meta_offset_le, ptr, sizeof(meta_offset_le));
+ meta_offset =
decode_fixed64_le(reinterpret_cast<uint8_t*>(&meta_offset_le));
+ ptr += sizeof(meta_offset_le);
+
+ // Deserialize checksum (convert from little-endian)
+ uint32_t checksum_le;
+ std::memcpy(&checksum_le, ptr, sizeof(checksum_le));
+ checksum = decode_fixed32_le(reinterpret_cast<uint8_t*>(&checksum_le));
+ ptr += sizeof(checksum_le);
+
+ version = *((uint8_t*)ptr);
+ ptr += sizeof(version);
+
+ // Deserialize magic
+ std::memcpy(magic, ptr, sizeof(magic));
+
+ return true;
+ }
+} __attribute__((packed));
+
+Status parse_dump_footer(std::ifstream& in, std::string& filename, size_t&
entry_num,
+ doris::io::cache::LRUDumpMetaPb& parse_meta,
+ doris::io::cache::LRUDumpEntryGroupPb&
current_parse_group) {
+ size_t file_size = std::filesystem::file_size(filename);
+
+ // Read footer
+ Footer footer;
+ size_t footer_size = sizeof(footer);
+ if (file_size < footer_size) {
+ std::string warn_msg = std::string(fmt::format(
+ "LRU dump file too small to contain footer, file={}, skip
restore", filename));
+ std::cerr << warn_msg << std::endl;
+ return Status::InternalError<false>(warn_msg);
+ }
+
+ in.seekg(-footer_size, std::ios::end);
+ std::string footer_str(footer_size, '\0');
+ in.read(&footer_str[0], footer_size);
+ RETURN_IF_ERROR(check_ifstream_status(in, filename));
+
+ if (!footer.deserialize_from_string(footer_str)) {
+ std::string warn_msg = std::string(
+ fmt::format("Failed to deserialize footer, file={}, skip
restore", filename));
+ std::cerr << warn_msg << std::endl;
+ return Status::InternalError<false>(warn_msg);
+ }
+
+ // Validate footer
+ if (footer.version != 1 || std::string(footer.magic, 3) != "DOR") {
+ std::string warn_msg = std::string(fmt::format(
+ "LRU dump file invalid footer format, file={}, skip restore",
filename));
+ std::cerr << warn_msg << std::endl;
+ return Status::InternalError<false>(warn_msg);
+ }
+
+ // Read meta
+ in.seekg(footer.meta_offset, std::ios::beg);
+ size_t meta_size = file_size - footer.meta_offset - footer_size;
+ if (meta_size <= 0) {
+ std::string warn_msg = std::string(
+ fmt::format("LRU dump file invalid meta size, file={}, skip
restore", filename));
+ std::cerr << warn_msg << std::endl;
+ return Status::InternalError<false>(warn_msg);
+ }
+ std::string meta_serialized(meta_size, '\0');
+ in.read(&meta_serialized[0], meta_serialized.size());
+ RETURN_IF_ERROR(check_ifstream_status(in, filename));
+ parse_meta.Clear();
+ current_parse_group.Clear();
+ if (!parse_meta.ParseFromString(meta_serialized)) {
+ std::string warn_msg = std::string(
+ fmt::format("LRU dump file meta parse failed, file={}, skip
restore", filename));
+ std::cerr << warn_msg << std::endl;
+ return Status::InternalError<false>(warn_msg);
+ }
+ std::cout << "parse meta: " << parse_meta.DebugString() << std::endl;
+
+ entry_num = parse_meta.entry_num();
+ return Status::OK();
+}
+
+Status parse_one_lru_entry(std::ifstream& in, std::string& filename,
io::UInt128Wrapper& hash,
+ size_t& offset, size_t& size,
+ doris::io::cache::LRUDumpMetaPb& parse_meta,
+ doris::io::cache::LRUDumpEntryGroupPb&
current_parse_group) {
+ // Read next group if current is empty
+ if (current_parse_group.entries_size() == 0) {
+ if (parse_meta.group_offset_size_size() == 0) {
+ return Status::EndOfFile("No more entries");
+ }
+
+ auto group_info = parse_meta.group_offset_size(0);
+ in.seekg(group_info.offset(), std::ios::beg);
+ std::string group_serialized(group_info.size(), '\0');
+ in.read(&group_serialized[0], group_serialized.size());
+ RETURN_IF_ERROR(check_ifstream_status(in, filename));
+ uint32_t checksum = crc32c::Value(group_serialized.data(),
group_serialized.size());
+ if (checksum != group_info.checksum()) {
+ std::string warn_msg =
+ fmt::format("restore lru failed as checksum not match,
file={}", filename);
+ std::cerr << warn_msg << std::endl;
+ return Status::InternalError(warn_msg);
+ }
+ if (!current_parse_group.ParseFromString(group_serialized)) {
+ std::string warn_msg =
+ fmt::format("restore lru failed to parse group, file={}",
filename);
+ std::cerr << warn_msg << std::endl;
+ return Status::InternalError(warn_msg);
+ }
+
+ // Remove processed group info
+
parse_meta.mutable_group_offset_size()->erase(parse_meta.group_offset_size().begin());
+ }
+
+ // Get next entry from current group
+ std::cout << "After deserialization: " <<
current_parse_group.DebugString() << std::endl;
+ auto entry = current_parse_group.entries(0);
+ hash = io::UInt128Wrapper((static_cast<uint128_t>(entry.hash().high()) <<
64) |
+ entry.hash().low());
+ offset = entry.offset();
+ size = entry.size();
+
+ std::cout << hash.to_string() << " " << offset << " " << size << std::endl;
+
+ // Remove processed entry
+
current_parse_group.mutable_entries()->erase(current_parse_group.entries().begin());
+ return Status::OK();
+}
+
+int main(int argc, char** argv) {
+ std::string usage = get_usage(argv[0]);
+ gflags::SetUsageMessage(usage);
+ google::ParseCommandLineFlags(&argc, &argv, true);
+
+ std::ifstream in(FLAGS_filename, std::ios::binary);
+ size_t entry_num;
+ doris::io::cache::LRUDumpMetaPb parse_meta;
+ doris::io::cache::LRUDumpEntryGroupPb current_parse_group;
+ auto s = parse_dump_footer(in, FLAGS_filename, entry_num, parse_meta,
current_parse_group);
+
+ in.seekg(0, std::ios::beg);
+ io::UInt128Wrapper hash;
+ size_t offset, size;
+ for (int i = 0; i < entry_num; ++i) {
+ EXIT_IF_ERROR(parse_one_lru_entry(in, FLAGS_filename, hash, offset,
size, parse_meta,
+ current_parse_group));
+ }
+
+ return 0;
+}
diff --git a/build.sh b/build.sh
index 543e03c2de3..fa80905d1ba 100755
--- a/build.sh
+++ b/build.sh
@@ -604,10 +604,15 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
BUILD_FS_BENCHMARK=OFF
fi
+ if [[ -z "${BUILD_FILE_CACHE_LRU_TOOL}" ]]; then
+ BUILD_FILE_CACHE_LRU_TOOL=OFF
+ fi
+
echo "-- Make program: ${MAKE_PROGRAM}"
echo "-- Use ccache: ${CMAKE_USE_CCACHE}"
echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS:-}"
echo "-- Build fs benchmark tool: ${BUILD_FS_BENCHMARK}"
+ echo "-- Build file cache lru tool: ${BUILD_FILE_CACHE_LRU_TOOL}"
mkdir -p "${CMAKE_BUILD_DIR}"
cd "${CMAKE_BUILD_DIR}"
@@ -619,6 +624,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
-DENABLE_CACHE_LOCK_DEBUG="${ENABLE_CACHE_LOCK_DEBUG}" \
-DMAKE_TEST=OFF \
-DBUILD_FS_BENCHMARK="${BUILD_FS_BENCHMARK}" \
+ -DBUILD_FILE_CACHE_LRU_TOOL="${BUILD_FILE_CACHE_LRU_TOOL}" \
${CMAKE_USE_CCACHE:+${CMAKE_USE_CCACHE}} \
-DWITH_MYSQL="${WITH_MYSQL}" \
-DUSE_LIBCPP="${USE_LIBCPP}" \
diff --git a/regression-test/suites/demo_p0/test_lru_persist.groovy
b/regression-test/suites/demo_p0/test_lru_persist.groovy
index 249faadeeda..d039c74c5c5 100644
--- a/regression-test/suites/demo_p0/test_lru_persist.groovy
+++ b/regression-test/suites/demo_p0/test_lru_persist.groovy
@@ -46,7 +46,6 @@ import org.apache.doris.regression.suite.ClusterOptions
suite('test_lru_persist', 'docker') {
def options = new ClusterOptions()
-
options.feNum = 1
options.beNum = 1
options.msNum = 1
@@ -90,5 +89,39 @@ suite('test_lru_persist', 'docker') {
logger.info("normalAfter: ${normalAfter}")
assert normalBefore == normalAfter
+
+ // remove dump file
+ def rm_dump_ret = "rm -rf
${cachePath}/lru_dump_normal.tail".execute().text.trim()
+ cluster.startBackends(1)
+ sleep(5000)
+ def show_backend_ret = sql '''show backends'''
+ try {
+ logger.info("Backend details: ${show_backend_ret.toString()}")
+ if(show_backend_ret.size() > 0 && show_backend_ret[0].size() > 0) {
+ logger.info("alive: ${show_backend_ret[0][9].toString()}")
+ }
+ assert show_backend_ret[0][9].toString() == "true"
+ } catch(Exception e) {
+ logger.error("Failed to log backend info: ${e.message}")
+ }
+
+ sql '''select count(*) from tb1'''
+
+ sleep(10000)
+ cluster.stopBackends(1)
+
+ def rm_data_ret = new File(cachePath).eachFile { file ->
+ if (!file.name.startsWith("lru_") && file.name != "version" &&
file.name != "." && file.name != "..") {
+ if (file.isDirectory()) {
+ file.deleteDir()
+ } else {
+ file.delete()
+ }
+ }
+ }
+ cluster.startBackends(1)
+ sleep(5000)
+
+ sql '''select count(*) from tb1'''
}
-}
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
