This is an automated email from the ASF dual-hosted git repository.
wzhou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 5b70e48eb IMPALA-13031: Enhancing logging for spilling configuration
with local buffer directory details
5b70e48eb is described below
commit 5b70e48ebbaac234338f2077ce6505f35b2d37aa
Author: Yida Wu <[email protected]>
AuthorDate: Tue Apr 23 18:14:25 2024 -0700
IMPALA-13031: Enhancing logging for spilling configuration with local
buffer directory details
The patch adds logging for local buffer directory when using
remote scratch space. The printed log would be like
"Using local buffer directory for scratch space
/tmp/test/impala-scratch on disk 8 limit: 500.00 MB,
priority: 2147483647".
Tests:
Manally tests the logging working as described.
Change-Id: I8fb357016d72a363ee5016f7881b0f6b0426aff5
Reviewed-on: http://gerrit.cloudera.org:8080/21350
Reviewed-by: Impala Public Jenkins <[email protected]>
Tested-by: Impala Public Jenkins <[email protected]>
---
be/src/runtime/tmp-file-mgr-internal.h | 4 ++++
be/src/runtime/tmp-file-mgr.cc | 14 ++++++++++----
2 files changed, 14 insertions(+), 4 deletions(-)
diff --git a/be/src/runtime/tmp-file-mgr-internal.h
b/be/src/runtime/tmp-file-mgr-internal.h
index db3076776..e800636f4 100644
--- a/be/src/runtime/tmp-file-mgr-internal.h
+++ b/be/src/runtime/tmp-file-mgr-internal.h
@@ -465,6 +465,10 @@ class TmpDirLocal : public TmpDir {
Status CreateLocalDirectory(MetricGroup* metrics, vector<bool>*
is_tmp_dir_on_disk,
bool need_local_buffer_dir, int disk_id, TmpFileMgr* tmp_mgr);
+ /// A helper function for CreateLocalDirectory() to log the configured local
directory
+ /// for scratch space.
+ void LogScratchLocalDirectoryInfo(bool is_local_buffer_dir, int disk_id);
+
Status ParsePathTokens(std::vector<string>& tokens) override;
};
diff --git a/be/src/runtime/tmp-file-mgr.cc b/be/src/runtime/tmp-file-mgr.cc
index c7039f776..522c42ba8 100644
--- a/be/src/runtime/tmp-file-mgr.cc
+++ b/be/src/runtime/tmp-file-mgr.cc
@@ -749,6 +749,14 @@ Status TmpDirLocal::VerifyAndCreate(MetricGroup* metrics,
return Status::OK();
}
+void TmpDirLocal::LogScratchLocalDirectoryInfo(bool is_local_buffer_dir, int
disk_id) {
+ LOG(INFO) << (is_local_buffer_dir ? "Using local buffer directory for
scratch space " :
+ "Using scratch directory ")
+ << path_ << " on "
+ << "disk " << disk_id << " limit: " <<
PrettyPrinter::PrintBytes(bytes_limit_)
+ << ", priority: " << priority_;
+}
+
Status TmpDirLocal::CreateLocalDirectory(MetricGroup* metrics,
vector<bool>* is_tmp_dir_on_disk, bool need_local_buffer_dir, int disk_id,
TmpFileMgr* tmp_mgr) {
@@ -771,13 +779,11 @@ Status TmpDirLocal::CreateLocalDirectory(MetricGroup*
metrics,
}
bytes_used_metric_ =
metrics->AddGauge(LOCAL_BUFF_BYTES_USED_FORMAT, 0, Substitute("$0",
0));
+ LogScratchLocalDirectoryInfo(true /*is_local_buffer_dir*/, disk_id);
return Status::OK();
}
if (disk_id >= 0) (*is_tmp_dir_on_disk)[disk_id] = true;
- LOG(INFO) << "Using scratch directory " << path_ << " on "
- << "disk " << disk_id
- << " limit: " << PrettyPrinter::PrintBytes(bytes_limit_)
- << ", priority: " << priority_;
+ LogScratchLocalDirectoryInfo(false /*is_local_buffer_dir*/, disk_id);
bytes_used_metric_ = metrics->AddGauge(
SCRATCH_DIR_BYTES_USED_FORMAT, 0, Substitute("$0",
tmp_mgr->tmp_dirs_.size()));
} else {
