This is an automated email from the ASF dual-hosted git repository.
rui pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d6e048a183 [VL] Enable Spark legacy date formatter (#7375)
d6e048a183 is described below
commit d6e048a183692e9c3185a46ffc52a798fb632c15
Author: Jaime Pan <[email protected]>
AuthorDate: Mon Oct 14 09:20:11 2024 +0800
[VL] Enable Spark legacy date formatter (#7375)
---
cpp/core/config/GlutenConfig.h | 2 ++
cpp/velox/compute/WholeStageResultIterator.cc | 7 +++++++
shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala | 2 ++
3 files changed, 11 insertions(+)
diff --git a/cpp/core/config/GlutenConfig.h b/cpp/core/config/GlutenConfig.h
index cd5196aa8c..47e5558f28 100644
--- a/cpp/core/config/GlutenConfig.h
+++ b/cpp/core/config/GlutenConfig.h
@@ -71,6 +71,8 @@ const std::string kIaaBackendName = "iaa";
const std::string kSparkRedactionRegex = "spark.redaction.regex";
const std::string kSparkRedactionString = "*********(redacted)";
+const std::string kSparkLegacyTimeParserPolicy =
"spark.sql.legacy.timeParserPolicy";
+
std::unordered_map<std::string, std::string>
parseConfMap(JNIEnv* env, const uint8_t* planData, const int32_t
planDataLength);
diff --git a/cpp/velox/compute/WholeStageResultIterator.cc
b/cpp/velox/compute/WholeStageResultIterator.cc
index 479aa9ff0f..8502790416 100644
--- a/cpp/velox/compute/WholeStageResultIterator.cc
+++ b/cpp/velox/compute/WholeStageResultIterator.cc
@@ -520,6 +520,13 @@ std::unordered_map<std::string, std::string>
WholeStageResultIterator::getQueryC
configs[velox::core::QueryConfig::kSparkPartitionId] =
std::to_string(taskInfo_.partitionId);
+ // Enable Spark legacy date formatter if spark.sql.legacy.timeParserPolicy
is set to 'LEGACY'.
+ if (veloxCfg_->get<std::string>(kSparkLegacyTimeParserPolicy, "") ==
"LEGACY") {
+ configs[velox::core::QueryConfig::kSparkLegacyDateFormatter] = "true";
+ } else {
+ configs[velox::core::QueryConfig::kSparkLegacyDateFormatter] = "false";
+ }
+
} catch (const std::invalid_argument& err) {
std::string errDetails = err.what();
throw std::runtime_error("Invalid conf arg: " + errDetails);
diff --git a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
index 7a384f7620..eaa5387512 100644
--- a/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
+++ b/shims/common/src/main/scala/org/apache/gluten/GlutenConfig.scala
@@ -665,6 +665,7 @@ object GlutenConfig {
SQLConf.SESSION_LOCAL_TIMEZONE.key,
GLUTEN_DEFAULT_SESSION_TIMEZONE_KEY,
SQLConf.LEGACY_SIZE_OF_NULL.key,
+ SQLConf.LEGACY_TIME_PARSER_POLICY.key,
"spark.io.compression.codec",
"spark.sql.decimalOperations.allowPrecisionLoss",
COLUMNAR_VELOX_BLOOM_FILTER_EXPECTED_NUM_ITEMS.key,
@@ -695,6 +696,7 @@ object GlutenConfig {
val keyWithDefault = ImmutableList.of(
(SQLConf.CASE_SENSITIVE.key, SQLConf.CASE_SENSITIVE.defaultValueString),
(SQLConf.IGNORE_MISSING_FILES.key,
SQLConf.IGNORE_MISSING_FILES.defaultValueString),
+ (SQLConf.LEGACY_TIME_PARSER_POLICY.key,
SQLConf.LEGACY_TIME_PARSER_POLICY.defaultValueString),
(
COLUMNAR_MEMORY_BACKTRACE_ALLOCATION.key,
COLUMNAR_MEMORY_BACKTRACE_ALLOCATION.defaultValueString),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
