This is an automated email from the ASF dual-hosted git repository.
ulyssesyou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new ae283e0e4 [VL] Add config for show velox task metrics when finished
(#6573)
ae283e0e4 is described below
commit ae283e0e42113bfdf6a7009f69d546d2724a8450
Author: Yang Zhang <[email protected]>
AuthorDate: Thu Jul 25 09:04:22 2024 +0800
[VL] Add config for show velox task metrics when finished (#6573)
---
cpp/velox/compute/WholeStageResultIterator.cc | 3 ++-
cpp/velox/config/VeloxConfig.h | 3 +++
shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala | 7 +++++++
3 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/cpp/velox/compute/WholeStageResultIterator.cc
b/cpp/velox/compute/WholeStageResultIterator.cc
index db8d51718..02125897c 100644
--- a/cpp/velox/compute/WholeStageResultIterator.cc
+++ b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -321,7 +321,8 @@ void WholeStageResultIterator::collectMetrics() {
return;
}
- if (veloxCfg_->get<bool>(kDebugModeEnabled, false)) {
+ if (veloxCfg_->get<bool>(kDebugModeEnabled, false) ||
+ veloxCfg_->get<bool>(kShowTaskMetricsWhenFinished,
kShowTaskMetricsWhenFinishedDefault)) {
auto planWithStats = velox::exec::printPlanWithStats(*veloxPlan_.get(),
task_->taskStats(), true);
std::ostringstream oss;
oss << "Native Plan with stats for: " << taskInfo_;
diff --git a/cpp/velox/config/VeloxConfig.h b/cpp/velox/config/VeloxConfig.h
index 7a96f03f4..65c7cb61d 100644
--- a/cpp/velox/config/VeloxConfig.h
+++ b/cpp/velox/config/VeloxConfig.h
@@ -59,6 +59,9 @@ const std::string kBloomFilterNumBits =
"spark.gluten.sql.columnar.backend.velox
const std::string kBloomFilterMaxNumBits =
"spark.gluten.sql.columnar.backend.velox.bloomFilter.maxNumBits";
const std::string kVeloxSplitPreloadPerDriver =
"spark.gluten.sql.columnar.backend.velox.SplitPreloadPerDriver";
+const std::string kShowTaskMetricsWhenFinished =
"spark.gluten.sql.columnar.backend.velox.showTaskMetricsWhenFinished";
+const bool kShowTaskMetricsWhenFinishedDefault = false;
+
const std::string kEnableUserExceptionStacktrace =
"spark.gluten.sql.columnar.backend.velox.enableUserExceptionStacktrace";
const bool kEnableUserExceptionStacktraceDefault = true;
diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
index b2aa176fd..5547feafe 100644
--- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -1710,6 +1710,13 @@ object GlutenConfig {
.booleanConf
.createWithDefault(true)
+ val COLUMNAR_VELOX_SHOW_TASK_METRICS_WHEN_FINISHED =
+
buildConf("spark.gluten.sql.columnar.backend.velox.showTaskMetricsWhenFinished")
+ .internal()
+ .doc("Show velox full task metrics when finished.")
+ .booleanConf
+ .createWithDefault(false)
+
val COLUMNAR_VELOX_MEMORY_USE_HUGE_PAGES =
buildConf("spark.gluten.sql.columnar.backend.velox.memoryUseHugePages")
.internal()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
