This is an automated email from the ASF dual-hosted git repository.
taiyangli pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 9837875fc6 [GLUTEN-7502][CH]Fix orc write time zone diff (#7523)
9837875fc6 is described below
commit 9837875fc68075b6459e4323c09e012a2143ecf4
Author: kevinyhzou <[email protected]>
AuthorDate: Mon Jan 6 16:13:28 2025 +0800
[GLUTEN-7502][CH]Fix orc write time zone diff (#7523)
* fix orc write diff
* add test
* fix ci
---
.../gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala | 10 ++++++++++
cpp-ch/local-engine/Storages/Output/ORCOutputFormatFile.cpp | 8 ++++++++
2 files changed, 18 insertions(+)
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala
index 0fee52108b..21289f35d4 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/hive/GlutenClickHouseHiveTableSuite.scala
@@ -1363,6 +1363,16 @@ class GlutenClickHouseHiveTableSuite
spark.sql("drop table test_tbl_6506")
}
+ test("GLUTEN-7502: Orc write time zone") {
+ val create_table_sql = "create table test_tbl_7502(id bigint, t timestamp)
using orc"
+ val insert_sql = "insert into test_tbl_7502 values(1, cast('2024-10-09
20:00:00' as timestamp))"
+ val select_sql = "select * from test_tbl_7502"
+ spark.sql(create_table_sql)
+ spark.sql(insert_sql);
+ compareResultsAgainstVanillaSpark(select_sql, compareResult = true, _ =>
{})
+ spark.sql("drop table test_tbl_7502")
+ }
+
test("GLUTEN-6879: Fix partition value diff when it contains blanks") {
val tableName = "test_tbl_6879"
sql(s"drop table if exists $tableName")
diff --git a/cpp-ch/local-engine/Storages/Output/ORCOutputFormatFile.cpp
b/cpp-ch/local-engine/Storages/Output/ORCOutputFormatFile.cpp
index b87b562fb1..0980814331 100644
--- a/cpp-ch/local-engine/Storages/Output/ORCOutputFormatFile.cpp
+++ b/cpp-ch/local-engine/Storages/Output/ORCOutputFormatFile.cpp
@@ -20,6 +20,8 @@
#if USE_ORC
# include <Formats/FormatFactory.h>
# include <Processors/Formats/Impl/ORCBlockOutputFormat.h>
+# include <Poco/Util/AbstractConfiguration.h>
+# include <Common/CHUtil.h>
namespace local_engine
{
@@ -40,6 +42,12 @@ OutputFormatFile::OutputFormatPtr
ORCOutputFormatFile::createOutputFormat(const
auto new_header = createHeaderWithPreferredSchema(header);
// TODO: align all spark orc config with ch orc config
auto format_settings = DB::getFormatSettings(context);
+ if (context->getConfigRef().has("timezone"))
+ {
+ const String config_timezone =
context->getConfigRef().getString("timezone");
+ const String mapped_timezone =
DateTimeUtil::convertTimeZone(config_timezone);
+ format_settings.orc.writer_time_zone_name = mapped_timezone;
+ }
auto output_format =
std::make_shared<DB::ORCBlockOutputFormat>(*(res->write_buffer), new_header,
format_settings);
res->output = output_format;
return res;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
