This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 51c2ebe5d87 branch-3.0: [fix](microbench) Fix some file cache
microbench compile bug #49269 (#49720)
51c2ebe5d87 is described below
commit 51c2ebe5d87266a44fa6d3c852799ead3990015e
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Apr 8 09:48:06 2025 +0800
branch-3.0: [fix](microbench) Fix some file cache microbench compile bug
#49269 (#49720)
Cherry-picked from #49269
Co-authored-by: deardeng <[email protected]>
---
be/CMakeLists.txt | 1 +
be/src/io/tools/file_cache_microbench.cpp | 27 +++++++++++++++++----------
build.sh | 2 +-
3 files changed, 19 insertions(+), 11 deletions(-)
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
index 4e3ea02ef47..1d5b3d86b7b 100644
--- a/be/CMakeLists.txt
+++ b/be/CMakeLists.txt
@@ -88,6 +88,7 @@ endif()
if (BUILD_FILE_CACHE_MICROBENCH_TOOL)
add_definitions(-DBE_TEST)
+ add_definitions(-DBUILD_FILE_CACHE_MICROBENCH_TOOL)
add_definitions(-DBE_BENCHMARK)
endif()
diff --git a/be/src/io/tools/file_cache_microbench.cpp
b/be/src/io/tools/file_cache_microbench.cpp
index 0111ff9f68c..6fda5de3474 100644
--- a/be/src/io/tools/file_cache_microbench.cpp
+++ b/be/src/io/tools/file_cache_microbench.cpp
@@ -284,9 +284,9 @@ private:
// Create a global S3FileRecords instance
S3FileRecords s3_file_records;
-class MircobenchS3FileWriter {
+class MicrobenchS3FileWriter {
public:
- MircobenchS3FileWriter(std::shared_ptr<doris::io::ObjClientHolder> client,
+ MicrobenchS3FileWriter(std::shared_ptr<doris::io::ObjClientHolder> client,
const std::string& bucket, const std::string& key,
const doris::io::FileWriterOptions* options,
std::shared_ptr<doris::S3RateLimiterHolder>
rate_limiter)
@@ -299,7 +299,7 @@ public:
}
using namespace doris;
if (write_bvar) {
- SCOPED_BVAR_LATENCY(*write_bvar)
+ SCOPED_BVAR_LATENCY(*write_bvar);
}
SCOPED_BVAR_LATENCY(microbench_write_latency);
return _writer.appendv(slices, slices_size);
@@ -326,7 +326,7 @@ public:
}
using namespace doris;
if (read_bvar) {
- SCOPED_BVAR_LATENCY(*read_bvar)
+ SCOPED_BVAR_LATENCY(*read_bvar);
}
SCOPED_BVAR_LATENCY(microbench_write_latency);
return _base_reader->read_at(offset, result, bytes_read, io_ctx);
@@ -399,14 +399,13 @@ public:
return res;
}
- ~ThreadPool() {
+ void stop_and_wait() {
{
std::unique_lock<std::mutex> lock(queue_mutex);
stop = true;
}
condition.notify_all();
- // Safely wait for all threads to complete
for (auto& worker : workers) {
try {
if (worker.joinable()) {
@@ -418,6 +417,16 @@ public:
}
}
+ ~ThreadPool() {
+ if (!stop) {
+ try {
+ stop_and_wait();
+ } catch (const std::exception& e) {
+ LOG(WARNING) << "Error stopping thread pool: " << e.what();
+ }
+ }
+ }
+
private:
std::vector<std::thread> workers;
std::queue<std::function<void()>> tasks;
@@ -793,7 +802,6 @@ private:
void init_limiters(const JobConfig& cfg) {
if (cfg.write_iops > 0) {
write_limiter = std::make_shared<doris::S3RateLimiterHolder>(
- doris::S3RateLimitType::PUT,
cfg.write_iops, // max_speed (IOPS)
cfg.write_iops, // max_burst
0, // no limit
@@ -806,7 +814,6 @@ private:
if (cfg.read_iops > 0) {
read_limiter = std::make_shared<doris::S3RateLimiterHolder>(
- doris::S3RateLimitType::GET,
cfg.read_iops, // max_speed (IOPS)
cfg.read_iops, // max_burst
0, // no limit
@@ -892,7 +899,7 @@ public:
void stop() {
LOG(INFO) << "Stopping JobManager and waiting for all jobs to
complete";
- _job_executor_pool.~ThreadPool();
+ _job_executor_pool.stop_and_wait();
LOG(INFO) << "JobManager stopped";
}
@@ -1061,7 +1068,7 @@ private:
options.file_cache_expiration = config.expiration;
}
options.write_file_cache = config.write_file_cache;
- auto writer = std::make_unique<MircobenchS3FileWriter>(
+ auto writer = std::make_unique<MicrobenchS3FileWriter>(
client, doris::config::test_s3_bucket, key,
&options,
job.write_limiter);
doris::Defer defer {[&]() {
diff --git a/build.sh b/build.sh
index 870311c109b..ada0745d27a 100755
--- a/build.sh
+++ b/build.sh
@@ -272,7 +272,7 @@ else
BUILD_CLOUD=1
BUILD_BROKER=1
BUILD_META_TOOL='ON'
- BUILD_FILE_CACHE_MICROBENCH_TOOL='ON'
+ BUILD_FILE_CACHE_MICROBENCH_TOOL='OFF'
BUILD_INDEX_TOOL='ON'
BUILD_SPARK_DPP=1
BUILD_HIVE_UDF=1
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
