This is an automated email from the ASF dual-hosted git repository.
yangzy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new fc8d37e8a9 [VL] Support pass unmapped dynamic and static configs to
Velox (#11229)
fc8d37e8a9 is described below
commit fc8d37e8a941925865d938084807703f1d53828b
Author: Yang Zhang <[email protected]>
AuthorDate: Wed Dec 3 11:12:32 2025 +0800
[VL] Support pass unmapped dynamic and static configs to Velox (#11229)
Avoid re-compile and publish whole project when we need overwrite some
configs in prod env.
---
cpp/velox/compute/WholeStageResultIterator.cc | 2 ++
cpp/velox/config/VeloxConfig.h | 3 +++
cpp/velox/utils/ConfigExtractor.cc | 14 ++++++++++++++
cpp/velox/utils/ConfigExtractor.h | 5 +++++
4 files changed, 24 insertions(+)
diff --git a/cpp/velox/compute/WholeStageResultIterator.cc
b/cpp/velox/compute/WholeStageResultIterator.cc
index da5d1c4c44..e1e8f7d00b 100644
--- a/cpp/velox/compute/WholeStageResultIterator.cc
+++ b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -663,6 +663,8 @@ std::unordered_map<std::string, std::string>
WholeStageResultIterator::getQueryC
setIfExists(kQueryTraceMaxBytes,
velox::core::QueryConfig::kQueryTraceMaxBytes);
setIfExists(kQueryTraceTaskRegExp,
velox::core::QueryConfig::kQueryTraceTaskRegExp);
setIfExists(kOpTraceDirectoryCreateConfig,
velox::core::QueryConfig::kOpTraceDirectoryCreateConfig);
+
+ overwriteVeloxConf(veloxCfg_.get(), configs, kDynamicBackendConfPrefix);
} catch (const std::invalid_argument& err) {
std::string errDetails = err.what();
throw std::runtime_error("Invalid conf arg: " + errDetails);
diff --git a/cpp/velox/config/VeloxConfig.h b/cpp/velox/config/VeloxConfig.h
index 2f3aad2187..419f6e79ca 100644
--- a/cpp/velox/config/VeloxConfig.h
+++ b/cpp/velox/config/VeloxConfig.h
@@ -193,4 +193,7 @@ const std::string kCudfEnableTableScan =
"spark.gluten.sql.columnar.backend.velo
const bool kCudfEnableTableScanDefault = false;
const std::string kCudfHiveConnectorId = "cudf-hive";
+const std::string kStaticBackendConfPrefix = "spark.gluten.velox.";
+const std::string kDynamicBackendConfPrefix =
"spark.gluten.sql.columnar.backend.velox.";
+
} // namespace gluten
diff --git a/cpp/velox/utils/ConfigExtractor.cc
b/cpp/velox/utils/ConfigExtractor.cc
index 5e4fa4ef2c..52e42d606d 100644
--- a/cpp/velox/utils/ConfigExtractor.cc
+++ b/cpp/velox/utils/ConfigExtractor.cc
@@ -237,6 +237,8 @@ std::shared_ptr<facebook::velox::config::ConfigBase>
createHiveConnectorSessionC
conf->get<bool>(kParquetUseColumnNames, true) ? "true" : "false";
configs[facebook::velox::connector::hive::HiveConfig::kOrcUseColumnNamesSession]
=
conf->get<bool>(kOrcUseColumnNames, true) ? "true" : "false";
+
+ overwriteVeloxConf(conf.get(), configs, kDynamicBackendConfPrefix);
return
std::make_shared<facebook::velox::config::ConfigBase>(std::move(configs));
}
@@ -299,7 +301,19 @@ std::shared_ptr<facebook::velox::config::ConfigBase>
createHiveConnectorConfig(
// read as UTC
hiveConfMap[facebook::velox::connector::hive::HiveConfig::kReadTimestampPartitionValueAsLocalTime]
= "false";
+ overwriteVeloxConf(conf.get(), hiveConfMap, kStaticBackendConfPrefix);
return
std::make_shared<facebook::velox::config::ConfigBase>(std::move(hiveConfMap));
}
+void overwriteVeloxConf(
+ const facebook::velox::config::ConfigBase* from,
+ std::unordered_map<std::string, std::string>& to,
+ const std::string& prefix) {
+ for (const auto& [k, v] : from->rawConfigs()) {
+ if (k.starts_with(prefix)) {
+ to[k.substr(prefix.size())] = v;
+ }
+ }
+}
+
} // namespace gluten
diff --git a/cpp/velox/utils/ConfigExtractor.h
b/cpp/velox/utils/ConfigExtractor.h
index cc7529c140..77b38c2f29 100644
--- a/cpp/velox/utils/ConfigExtractor.h
+++ b/cpp/velox/utils/ConfigExtractor.h
@@ -43,4 +43,9 @@ std::shared_ptr<facebook::velox::config::ConfigBase>
createHiveConnectorConfig(
const std::shared_ptr<facebook::velox::config::ConfigBase>& conf,
FileSystemType fsType = FileSystemType::kAll);
+void overwriteVeloxConf(
+ const facebook::velox::config::ConfigBase* from,
+ std::unordered_map<std::string, std::string>& to,
+ const std::string& prefix);
+
} // namespace gluten
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
