This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new d8bd3cd0c2 [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250326)
(#9136)
d8bd3cd0c2 is described below
commit d8bd3cd0c224e3c9b403bd24e6cf3fc9e7f2a8e9
Author: Kyligence Git <[email protected]>
AuthorDate: Wed Mar 26 09:41:37 2025 -0500
[GLUTEN-1632][CH]Daily Update Clickhouse Version (20250326) (#9136)
* [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250326)
* Fix build due to https://github.com/ClickHouse/ClickHouse/pull/55518
* Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/77013
* Fix gtest build due to https://github.com/ClickHouse/ClickHouse/pull/77895
* disbable query_plan_optimize_lazy_materialization due to
https://github.com/apache/incubator-gluten/issues/9141
---------
Co-authored-by: kyligence-git <[email protected]>
Co-authored-by: Chang chen <[email protected]>
---
cpp-ch/clickhouse.version | 4 ++--
cpp-ch/local-engine/Common/CHUtil.cpp | 6 ++++++
cpp-ch/local-engine/Parser/SerializedPlanParser.cpp | 4 +++-
cpp-ch/local-engine/Storages/IO/NativeReader.cpp | 4 ++--
cpp-ch/local-engine/tests/gtest_local_engine.cpp | 2 +-
5 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
index 6013089432..aa4332e58a 100644
--- a/cpp-ch/clickhouse.version
+++ b/cpp-ch/clickhouse.version
@@ -1,3 +1,3 @@
CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20250320
-CH_COMMIT=71209145bad
+CH_BRANCH=rebase_ch/20250326
+CH_COMMIT=16822e629e2
diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp
b/cpp-ch/local-engine/Common/CHUtil.cpp
index adbcfee8f8..3cef2e18b2 100644
--- a/cpp-ch/local-engine/Common/CHUtil.cpp
+++ b/cpp-ch/local-engine/Common/CHUtil.cpp
@@ -103,6 +103,7 @@ extern const SettingsBool query_plan_merge_filters;
extern const SettingsBool compile_expressions;
extern const SettingsShortCircuitFunctionEvaluation
short_circuit_function_evaluation;
extern const SettingsUInt64 output_format_compression_level;
+extern const SettingsBool query_plan_optimize_lazy_materialization;
}
namespace ErrorCodes
{
@@ -692,6 +693,11 @@ void BackendInitializerUtil::initSettings(const
SparkConfigs::ConfigMap & spark_
/// output_format_compression_level is set to 3, which is wrong, since
snappy does not support it.
settings[Setting::output_format_compression_level] =
arrow::util::kUseDefaultCompressionLevel;
+ /// 6. After https://github.com/ClickHouse/ClickHouse/pull/55518
+ /// We currently do not support lazy materialization.
+ /// "test 'order by' two keys" will failed if we enable it.
+ settings[Setting::query_plan_optimize_lazy_materialization] = false;
+
for (const auto & [key, value] : spark_conf_map)
{
// Firstly apply
spark.gluten.sql.columnar.backend.ch.runtime_config.local_engine.settings.* to
settings
diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp
b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp
index 4cc81f5384..4eee44400d 100644
--- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp
+++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp
@@ -65,6 +65,7 @@ namespace DB
namespace Setting
{
extern const SettingsUInt64 priority;
+extern const SettingsMilliseconds low_priority_query_wait_time_ms;
}
namespace ErrorCodes
{
@@ -304,7 +305,8 @@ DB::QueryPipelineBuilderPtr
SerializedPlanParser::buildQueryPipeline(DB::QueryPl
"",
0, // since we set a query to empty string, let's set hash to zero.
parser_context->queryContext()->getClientInfo(),
- priorities.insert(settings[Setting::priority]),
+ priorities.insert(
+ settings[Setting::priority],
std::chrono::milliseconds(settings[Setting::low_priority_query_wait_time_ms].totalMilliseconds())),
CurrentThread::getGroup(),
IAST::QueryKind::Select,
settings,
diff --git a/cpp-ch/local-engine/Storages/IO/NativeReader.cpp
b/cpp-ch/local-engine/Storages/IO/NativeReader.cpp
index 65c2ad8af9..7b016eb886 100644
--- a/cpp-ch/local-engine/Storages/IO/NativeReader.cpp
+++ b/cpp-ch/local-engine/Storages/IO/NativeReader.cpp
@@ -130,7 +130,7 @@ static void readNormalSimpleData(DB::ReadBuffer &in,
DB::ColumnPtr & column, siz
ISerialization::DeserializeBinaryBulkStatePtr state;
column_parse_util.serializer->deserializeBinaryBulkStatePrefix(settings,
state, nullptr);
-
column_parse_util.serializer->deserializeBinaryBulkWithMultipleStreams(column,
rows, settings, state, nullptr);
+
column_parse_util.serializer->deserializeBinaryBulkWithMultipleStreams(column,
0, rows, settings, state, nullptr);
}
// May not efficient.
@@ -147,7 +147,7 @@ readNormalComplexData(DB::ReadBuffer & in, DB::ColumnPtr &
column, size_t rows,
DB::ColumnPtr new_col = column->cloneResized(0);
column_parse_util.serializer->deserializeBinaryBulkStatePrefix(settings,
state ,nullptr);
-
column_parse_util.serializer->deserializeBinaryBulkWithMultipleStreams(new_col,
rows, settings, state, nullptr);
+
column_parse_util.serializer->deserializeBinaryBulkWithMultipleStreams(new_col,
0, rows, settings, state, nullptr);
column->assumeMutable()->insertRangeFrom(*new_col, 0, new_col->size());
}
diff --git a/cpp-ch/local-engine/tests/gtest_local_engine.cpp
b/cpp-ch/local-engine/tests/gtest_local_engine.cpp
index 40321cfda3..2cffbdb8c5 100644
--- a/cpp-ch/local-engine/tests/gtest_local_engine.cpp
+++ b/cpp-ch/local-engine/tests/gtest_local_engine.cpp
@@ -106,7 +106,7 @@ int main(int argc, char ** argv)
SCOPE_EXIT({ BackendFinalizerUtil::finalizeGlobally(); });
DB::registerInterpreters();
- DB::registerTableFunctions(false);
+ DB::registerTableFunctions();
::testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
