This is an automated email from the ASF dual-hosted git repository.
rui pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d36b76957 [VL] Fix kParquetWriteTimestampUnit to
kParquetWriteTimestampUnitSession (#5281)
d36b76957 is described below
commit d36b76957cf2133d4ca801603808e4c15d0c759d
Author: Yang Zhang <[email protected]>
AuthorDate: Thu Apr 11 09:05:05 2024 +0800
[VL] Fix kParquetWriteTimestampUnit to kParquetWriteTimestampUnitSession
(#5281)
---
cpp/velox/compute/VeloxBackend.cc | 4 ++--
cpp/velox/compute/WholeStageResultIterator.cc | 3 ++-
2 files changed, 4 insertions(+), 3 deletions(-)
diff --git a/cpp/velox/compute/VeloxBackend.cc
b/cpp/velox/compute/VeloxBackend.cc
index 6e010ec18..8f1cab48b 100644
--- a/cpp/velox/compute/VeloxBackend.cc
+++ b/cpp/velox/compute/VeloxBackend.cc
@@ -258,8 +258,7 @@ void VeloxBackend::initCache(const std::shared_ptr<const
facebook::velox::Config
}
void VeloxBackend::initConnector(const std::shared_ptr<const
facebook::velox::Config>& conf) {
- int32_t ioThreads = conf->get<int32_t>(kVeloxIOThreads,
kVeloxIOThreadsDefault);
-
+ // The configs below are used at process level.
auto mutableConf =
std::make_shared<facebook::velox::core::MemConfigMutable>(conf->valuesCopy());
auto hiveConf = getHiveConfig(conf);
@@ -303,6 +302,7 @@ void VeloxBackend::initConnector(const
std::shared_ptr<const facebook::velox::Co
// set cache_prefetch_min_pct default as 0 to force all loads are prefetched
in DirectBufferInput.
FLAGS_cache_prefetch_min_pct = conf->get<int>(kCachePrefetchMinPct, 0);
+ auto ioThreads = conf->get<int32_t>(kVeloxIOThreads, kVeloxIOThreadsDefault);
if (ioThreads > 0) {
ioExecutor_ = std::make_unique<folly::IOThreadPoolExecutor>(ioThreads);
}
diff --git a/cpp/velox/compute/WholeStageResultIterator.cc
b/cpp/velox/compute/WholeStageResultIterator.cc
index 89b77ac85..e105a0d64 100644
--- a/cpp/velox/compute/WholeStageResultIterator.cc
+++ b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -559,12 +559,13 @@ std::unordered_map<std::string, std::string>
WholeStageResultIterator::getQueryC
}
std::shared_ptr<velox::Config>
WholeStageResultIterator::createConnectorConfig() {
+ // The configs below are used at session level.
std::unordered_map<std::string, std::string> configs = {};
// The semantics of reading as lower case is opposite with case-sensitive.
configs[velox::connector::hive::HiveConfig::kFileColumnNamesReadAsLowerCaseSession]
=
!veloxCfg_->get<bool>(kCaseSensitive, false) ? "true" : "false";
configs[velox::connector::hive::HiveConfig::kPartitionPathAsLowerCaseSession] =
"false";
- configs[velox::connector::hive::HiveConfig::kParquetWriteTimestampUnit] =
"6";
+
configs[velox::connector::hive::HiveConfig::kParquetWriteTimestampUnitSession]
= "6";
configs[velox::connector::hive::HiveConfig::kMaxPartitionsPerWritersSession]
=
std::to_string(veloxCfg_->get<int32_t>(kMaxPartitions, 10000));
configs[velox::connector::hive::HiveConfig::kIgnoreMissingFilesSession] =
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
