This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new c14d1b12b6 [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250103)
(#8407)
c14d1b12b6 is described below
commit c14d1b12b6795eb31e2a5b0f14df821ab8035089
Author: Kyligence Git <[email protected]>
AuthorDate: Fri Jan 3 08:19:14 2025 -0600
[GLUTEN-1632][CH]Daily Update Clickhouse Version (20250103) (#8407)
* [GLUTEN-1632][CH]Daily Update Clickhouse Version (20250103)
* Fix gtest due to https://github.com/ClickHouse/ClickHouse/pull/73811
* Another way to set query_plan_enable_optimizations to false
* Try to set max_threads to 1, but
GlutenClickHouseTPCHColumnarShuffleParquetAQESuite::q1 and q2 will fail
* Fix Build Due to https://github.com/ClickHouse/ClickHouse/pull/73693
* Restore QueryPlanOptimizationSettings
* COLLECT_METRICS
* Fix build due to https://github.com/ClickHouse/ClickHouse/pull/73994
---------
Co-authored-by: kyligence-git <[email protected]>
Co-authored-by: Chang Chen <[email protected]>
---
.../backendsapi/clickhouse/CHTransformerApi.scala | 7 -------
.../gluten/backendsapi/clickhouse/RuntimeSettings.scala | 11 +++++++++++
.../tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala | 7 +++----
cpp-ch/clickhouse.version | 4 ++--
cpp-ch/local-engine/Common/CHUtil.cpp | 4 ++++
cpp-ch/local-engine/Common/GlutenConfig.h | 4 ++--
cpp-ch/local-engine/Common/GlutenSettings.cpp | 6 +++++-
cpp-ch/local-engine/Common/GlutenSettings.h | 11 ++++++++++-
cpp-ch/local-engine/Common/PlanUtil.cpp | 2 +-
cpp-ch/local-engine/Operator/BranchStep.cpp | 17 +++++++++--------
cpp-ch/local-engine/Operator/BranchStep.h | 6 ++++--
.../Parser/RelParsers/GroupLimitRelParser.cpp | 2 +-
cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp | 4 ++--
cpp-ch/local-engine/Parser/SerializedPlanParser.cpp | 13 +++++++++----
.../Parser/scalar_function_parser/arrayExcept.cpp | 2 +-
.../Parser/scalar_function_parser/arrayRemove.cpp | 2 +-
.../Parser/scalar_function_parser/arrayRepeat.cpp | 2 +-
.../Parser/scalar_function_parser/lambdaFunction.cpp | 2 +-
cpp-ch/local-engine/Storages/Cache/CacheManager.cpp | 4 +++-
cpp-ch/local-engine/tests/gtest_ch_functions.cpp | 6 +++---
cpp-ch/local-engine/tests/gtest_ch_join.cpp | 7 ++++---
.../tests/json/gtest_local_engine_config.json | 1 -
22 files changed, 77 insertions(+), 47 deletions(-)
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala
index ef5a4eff6f..8487388f3f 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHTransformerApi.scala
@@ -142,13 +142,6 @@ class CHTransformerApi extends TransformerApi with Logging
{
"spark.hadoop.dfs.client.log.severity",
s"$hdfsConfigPrefix.dfs_client_log_severity")
- // TODO: set default to true when metrics could be collected
- // while ch query plan optimization is enabled.
- val planOptKey = CHConf.runtimeSettings("query_plan_enable_optimizations")
- if (!nativeConfMap.containsKey(planOptKey)) {
- nativeConfMap.put(planOptKey, "false")
- }
-
// Respect spark config spark.sql.orc.compression.codec for CH backend
// TODO: consider compression or orc.compression in table options.
val orcCompressionKey =
CHConf.runtimeSettings("output_format_orc_compression_method")
diff --git
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/RuntimeSettings.scala
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/RuntimeSettings.scala
index 140b09a9ab..4365b2987c 100644
---
a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/RuntimeSettings.scala
+++
b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/RuntimeSettings.scala
@@ -39,6 +39,17 @@ object RuntimeSettings {
// scalastyle:on line.size.limit
/** Gluten Configuration */
+ // scalastyle:off line.size.limit
+ val COLLECT_METRICS =
+ buildConf(runtimeSettings("collect_metrics"))
+ .doc(s"""If true, we need disable query_plan_enable_optimizations,
otherwise clickhouse optimize the query plan
+ |and cause collecting metrics failed.
+ |see
https://clickhouse.com/docs/en/operations/settings/settings#query_plan_enable_optimizations
+ |""".stripMargin)
+ .booleanConf
+ .createWithDefault(true)
+ // scalastyle:on line.size.limit
+
val NATIVE_WRITE_RESERVE_PARTITION_COLUMNS =
buildConf(runtimeSettings("gluten.write.reserve_partition_columns"))
.doc("Whether reserve partition columns for Native write or not, default
is false")
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala
index 6feb1e8806..361797c97d 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/tpch/GlutenClickHouseTPCHSaltNullParquetSuite.scala
@@ -17,7 +17,7 @@
package org.apache.gluten.execution.tpch
import org.apache.gluten.GlutenConfig
-import org.apache.gluten.backendsapi.clickhouse.CHConf
+import org.apache.gluten.backendsapi.clickhouse.{CHConf, RuntimeSettings}
import org.apache.gluten.execution._
import org.apache.gluten.execution.GlutenPlan
@@ -1419,12 +1419,11 @@ class GlutenClickHouseTPCHSaltNullParquetSuite extends
GlutenClickHouseTPCHAbstr
queriesResults: String = queriesResults,
compareResult: Boolean = true,
noFallBack: Boolean = true)(customCheck: DataFrame => Unit): Unit = {
- val confName = CHConf.runtimeSettings("query_plan_enable_optimizations")
- withSQLConf((confName, "true")) {
+ withSQLConf((RuntimeSettings.COLLECT_METRICS.key, "false")) {
compareTPCHQueryAgainstVanillaSpark(queryNum, tpchQueries, customCheck,
noFallBack)
}
- withSQLConf((confName, "false")) {
+ withSQLConf((RuntimeSettings.COLLECT_METRICS.key, "true")) {
compareTPCHQueryAgainstVanillaSpark(queryNum, tpchQueries, customCheck,
noFallBack)
}
}
diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
index 83bbbf9a21..87fefef5cc 100644
--- a/cpp-ch/clickhouse.version
+++ b/cpp-ch/clickhouse.version
@@ -1,3 +1,3 @@
CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20241228
-CH_COMMIT=bf8e58b57e9
+CH_BRANCH=rebase_ch/20250103
+CH_COMMIT=0523d6de564
diff --git a/cpp-ch/local-engine/Common/CHUtil.cpp
b/cpp-ch/local-engine/Common/CHUtil.cpp
index c85104e5ae..ace3a515a2 100644
--- a/cpp-ch/local-engine/Common/CHUtil.cpp
+++ b/cpp-ch/local-engine/Common/CHUtil.cpp
@@ -80,6 +80,7 @@ extern const ServerSettingsDouble
primary_index_cache_size_ratio;
}
namespace Setting
{
+extern const SettingsMaxThreads max_threads;
extern const SettingsUInt64 prefer_external_sort_block_bytes;
extern const SettingsUInt64 max_bytes_before_external_sort;
extern const SettingsDouble max_bytes_ratio_before_external_sort;
@@ -635,6 +636,9 @@ void BackendInitializerUtil::initSettings(const
SparkConfigs::ConfigMap & spark_
settings.set("input_format_parquet_enable_row_group_prefetch", false);
settings.set("output_format_parquet_use_custom_encoder", false);
+ // TODO: we need set Setting::max_threads to 1 by default, but now we
can't get correct metrics for the some query if we set it to 1.
+ // settings[Setting::max_threads] = 1;
+
/// Set false after https://github.com/ClickHouse/ClickHouse/pull/71539
/// if true, we can't get correct metrics for the query
settings[Setting::query_plan_merge_filters] = false;
diff --git a/cpp-ch/local-engine/Common/GlutenConfig.h
b/cpp-ch/local-engine/Common/GlutenConfig.h
index c0fa53e630..1aefca1bc4 100644
--- a/cpp-ch/local-engine/Common/GlutenConfig.h
+++ b/cpp-ch/local-engine/Common/GlutenConfig.h
@@ -172,8 +172,8 @@ public:
namespace PathConfig
{
-inline constexpr const char * USE_CURRENT_DIRECTORY_AS_TMP =
"use_current_directory_as_tmp";
-inline constexpr const char * DEFAULT_TEMP_FILE_PATH = "/tmp/libch";
+inline constexpr auto USE_CURRENT_DIRECTORY_AS_TMP =
"use_current_directory_as_tmp";
+inline constexpr auto DEFAULT_TEMP_FILE_PATH = "/tmp/libch";
};
/// Configurations for spark.sql.
diff --git a/cpp-ch/local-engine/Common/GlutenSettings.cpp
b/cpp-ch/local-engine/Common/GlutenSettings.cpp
index ab015b5636..42451c3ed8 100644
--- a/cpp-ch/local-engine/Common/GlutenSettings.cpp
+++ b/cpp-ch/local-engine/Common/GlutenSettings.cpp
@@ -18,6 +18,7 @@
#include <Core/Settings.h>
#include <Interpreters/Context.h>
+#include <Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h>
#include <Common/GlutenConfig.h>
using namespace DB;
@@ -33,10 +34,13 @@ bool tryGetString(const DB::Settings & settings,
std::string_view name, std::str
}
return false;
}
-bool settingsEqual(const DB::Settings & settings, std::string_view name, const
std::string & value)
+bool settingsEqual(
+ const DB::Settings & settings, std::string_view name, const std::string &
value, const std::optional<std::string> & default_value)
{
if (DB::Field field; settings.tryGet(name, field))
return field.safeGet<String>() == value;
+ if (default_value.has_value())
+ return *default_value == value;
return false;
}
void updateSettings(const DB::ContextMutablePtr & context, std::string_view
plan)
diff --git a/cpp-ch/local-engine/Common/GlutenSettings.h
b/cpp-ch/local-engine/Common/GlutenSettings.h
index 99c01368b4..153bb7c6b5 100644
--- a/cpp-ch/local-engine/Common/GlutenSettings.h
+++ b/cpp-ch/local-engine/Common/GlutenSettings.h
@@ -67,7 +67,16 @@ namespace local_engine
// workaround for tryGetString
bool tryGetString(const DB::Settings & settings, std::string_view name,
std::string & value);
-bool settingsEqual(const DB::Settings & settings, std::string_view name, const
std::string & value);
+bool settingsEqual(
+ const DB::Settings & settings,
+ std::string_view name,
+ const std::string & value,
+ const std::optional<std::string> & default_value = std::nullopt);
void updateSettings(const DB::ContextMutablePtr &, std::string_view);
+namespace RuntimeSettings
+{
+inline constexpr auto COLLECT_METRICS = "collect_metrics";
+inline constexpr auto COLLECT_METRICS_DEFAULT = "true";
+}
} // namespace local_engine
diff --git a/cpp-ch/local-engine/Common/PlanUtil.cpp
b/cpp-ch/local-engine/Common/PlanUtil.cpp
index 948cf35a5a..ebc3b6da3d 100644
--- a/cpp-ch/local-engine/Common/PlanUtil.cpp
+++ b/cpp-ch/local-engine/Common/PlanUtil.cpp
@@ -36,7 +36,7 @@ namespace local_engine::PlanUtil
std::string explainPlan(const DB::QueryPlan & plan)
{
- constexpr DB::QueryPlan::ExplainPlanOptions buf_opt{
+ constexpr DB::ExplainPlanOptions buf_opt{
.header = true,
.actions = true,
.indexes = true,
diff --git a/cpp-ch/local-engine/Operator/BranchStep.cpp
b/cpp-ch/local-engine/Operator/BranchStep.cpp
index 5e379ae9d4..c52b14420c 100644
--- a/cpp-ch/local-engine/Operator/BranchStep.cpp
+++ b/cpp-ch/local-engine/Operator/BranchStep.cpp
@@ -137,7 +137,7 @@ public:
private:
size_t num_streams;
- void updateOutputHeader() override {};
+ void updateOutputHeader() override { };
};
DB::QueryPlanPtr BranchStepHelper::createSubPlan(const DB::Block & header,
size_t num_streams)
@@ -156,7 +156,7 @@ DB::QueryPlanPtr BranchStepHelper::createSubPlan(const
DB::Block & header, size_
}
StaticBranchStep::StaticBranchStep(
- DB::ContextPtr context_, const DB::Block & header_, size_t branches_,
size_t sample_rows_, BranchSelector selector_)
+ const DB::ContextPtr & context_, const DB::Block & header_, size_t
branches_, size_t sample_rows_, BranchSelector selector_)
: DB::ITransformingStep(header_, header_, getTraits())
, context(context_)
, header(header_)
@@ -168,7 +168,7 @@ StaticBranchStep::StaticBranchStep(
void StaticBranchStep::transformPipeline(DB::QueryPipelineBuilder & pipeline,
const DB::BuildQueryPipelineSettings & settings)
{
- auto build_transform = [&](DB::OutputPortRawPtrs child_outputs) ->
DB::Processors
+ auto build_transform = [&](const DB::OutputPortRawPtrs & child_outputs) ->
DB::Processors
{
DB::Processors new_processors;
for (auto & output : child_outputs)
@@ -195,8 +195,9 @@ void StaticBranchStep::updateOutputHeader()
{
}
-UniteBranchesStep::UniteBranchesStep(const DB::Block & header_,
std::vector<DB::QueryPlanPtr> && branch_plans_, size_t num_streams_)
- : DB::ITransformingStep(header_, branch_plans_[0]->getCurrentHeader(),
getTraits()), header(header_)
+UniteBranchesStep::UniteBranchesStep(
+ const DB::ContextPtr & context_, const DB::Block & header_,
std::vector<DB::QueryPlanPtr> && branch_plans_, size_t num_streams_)
+ : DB::ITransformingStep(header_, branch_plans_[0]->getCurrentHeader(),
getTraits()), context(context_), header(header_)
{
branch_plans.swap(branch_plans_);
size_t branches = branch_plans.size();
@@ -217,11 +218,11 @@ void
UniteBranchesStep::transformPipeline(DB::QueryPipelineBuilder & pipeline, c
child_outputs.size(),
branch_plans.size());
}
- for (auto output : child_outputs)
+ for (auto * output : child_outputs)
{
auto & branch_plan = branch_plans[branch_index];
- DB::QueryPlanOptimizationSettings optimization_settings;
- DB::BuildQueryPipelineSettings build_settings;
+ DB::QueryPlanOptimizationSettings optimization_settings{context};
+ DB::BuildQueryPipelineSettings build_settings{context};
DB::QueryPlanResourceHolder resource_holder;
auto pipeline_builder =
branch_plan->buildQueryPipeline(optimization_settings, build_settings);
diff --git a/cpp-ch/local-engine/Operator/BranchStep.h
b/cpp-ch/local-engine/Operator/BranchStep.h
index fd2203aae8..ef3d69e5c6 100644
--- a/cpp-ch/local-engine/Operator/BranchStep.h
+++ b/cpp-ch/local-engine/Operator/BranchStep.h
@@ -42,7 +42,7 @@ class StaticBranchStep : public DB::ITransformingStep
public:
using BranchSelector = std::function<size_t(const std::list<DB::Chunk> &)>;
explicit StaticBranchStep(
- DB::ContextPtr context_, const DB::Block & header, size_t branches,
size_t sample_rows, BranchSelector selector);
+ const DB::ContextPtr & context_, const DB::Block & header, size_t
branches, size_t sample_rows, BranchSelector selector);
~StaticBranchStep() override = default;
String getName() const override { return "StaticBranchStep"; }
@@ -67,7 +67,8 @@ private:
class UniteBranchesStep : public DB::ITransformingStep
{
public:
- explicit UniteBranchesStep(const DB::Block & header_,
std::vector<DB::QueryPlanPtr> && branch_plans_, size_t num_streams_);
+ explicit UniteBranchesStep(
+ const DB::ContextPtr & context_, const DB::Block & header_,
std::vector<DB::QueryPlanPtr> && branch_plans_, size_t num_streams_);
~UniteBranchesStep() override = default;
String getName() const override { return "UniteBranchesStep"; }
@@ -76,6 +77,7 @@ public:
void describePipeline(DB::IQueryPlanStep::FormatSettings & settings) const
override;
private:
+ DB::ContextPtr context;
DB::Block header;
std::vector<DB::QueryPlanPtr> branch_plans;
size_t num_streams;
diff --git a/cpp-ch/local-engine/Parser/RelParsers/GroupLimitRelParser.cpp
b/cpp-ch/local-engine/Parser/RelParsers/GroupLimitRelParser.cpp
index 8306346827..b43c8064b3 100644
--- a/cpp-ch/local-engine/Parser/RelParsers/GroupLimitRelParser.cpp
+++ b/cpp-ch/local-engine/Parser/RelParsers/GroupLimitRelParser.cpp
@@ -245,7 +245,7 @@ DB::QueryPlanPtr AggregateGroupLimitRelParser::parse(
std::vector<DB::QueryPlanPtr> branch_plans;
branch_plans.emplace_back(std::move(aggregation_plan));
branch_plans.emplace_back(std::move(window_plan));
- auto unite_branches_step =
std::make_unique<UniteBranchesStep>(branch_in_header, std::move(branch_plans),
1);
+ auto unite_branches_step =
std::make_unique<UniteBranchesStep>(getContext(), branch_in_header,
std::move(branch_plans), 1);
unite_branches_step->setStepDescription("Unite TopK branches");
steps.push_back(unite_branches_step.get());
diff --git a/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp
b/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp
index 7c5aeb23b9..3db111b6d4 100644
--- a/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp
+++ b/cpp-ch/local-engine/Parser/RelParsers/JoinRelParser.cpp
@@ -599,8 +599,8 @@ bool JoinRelParser::applyJoinFilter(
return false;
auto mixed_join_expressions_actions = expressionsToActionsDAG({expr},
mixed_header);
mixed_join_expressions_actions.removeUnusedActions();
- table_join.getMixedJoinExpression() =
std::make_shared<DB::ExpressionActions>(
- std::move(mixed_join_expressions_actions),
ExpressionActionsSettings::fromContext(context));
+ table_join.getMixedJoinExpression()
+ =
std::make_shared<DB::ExpressionActions>(std::move(mixed_join_expressions_actions),
ExpressionActionsSettings(context));
}
else
{
diff --git a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp
b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp
index f3a9c3c901..e1733e915a 100644
--- a/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp
+++ b/cpp-ch/local-engine/Parser/SerializedPlanParser.cpp
@@ -64,7 +64,6 @@ namespace DB
{
namespace Setting
{
-extern const SettingsBool query_plan_enable_optimizations;
extern const SettingsUInt64 priority;
}
namespace ErrorCodes
@@ -276,7 +275,8 @@ QueryPlanPtr SerializedPlanParser::parseOp(const
substrait::Rel & rel, std::list
std::vector<DB::IQueryPlanStep *> steps = rel_parser->getSteps();
- if
(!parser_context->queryContext()->getSettingsRef()[Setting::query_plan_enable_optimizations])
+ if (const auto & settings =
parser_context->queryContext()->getSettingsRef();
+ settingsEqual(settings, RuntimeSettings::COLLECT_METRICS, "true",
{RuntimeSettings::COLLECT_METRICS_DEFAULT}))
{
if (rel.rel_type_case() == substrait::Rel::RelTypeCase::kRead)
{
@@ -303,8 +303,13 @@ DB::QueryPipelineBuilderPtr
SerializedPlanParser::buildQueryPipeline(DB::QueryPl
IAST::QueryKind::Select,
settings,
0);
- const QueryPlanOptimizationSettings optimization_settings{.optimize_plan =
settings[Setting::query_plan_enable_optimizations]};
- BuildQueryPipelineSettings build_settings =
BuildQueryPipelineSettings::fromContext(context);
+ QueryPlanOptimizationSettings optimization_settings{context};
+
+ // TODO: set optimize_plan to true when metrics could be collected while
ch query plan optimization is enabled.
+ if (settingsEqual(settings, RuntimeSettings::COLLECT_METRICS, "true",
{RuntimeSettings::COLLECT_METRICS_DEFAULT}))
+ optimization_settings.optimize_plan = false;
+
+ BuildQueryPipelineSettings build_settings =
BuildQueryPipelineSettings{context};
build_settings.process_list_element = query_status;
build_settings.progress_callback = nullptr;
return query_plan.buildQueryPipeline(optimization_settings,
build_settings);
diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayExcept.cpp
b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayExcept.cpp
index 18b4f69aa4..e041985947 100644
--- a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayExcept.cpp
+++ b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayExcept.cpp
@@ -74,7 +74,7 @@ public:
NamesAndTypesList lambda_arguments_names_and_types;
lambda_arguments_names_and_types.emplace_back(x_in_lambda->result_name,
x_in_lambda->result_type);
DB::Names required_column_names =
lambda_actions_dag.getRequiredColumnsNames();
- auto expression_actions_settings =
DB::ExpressionActionsSettings::fromContext(getContext(),
DB::CompileExpressions::yes);
+ auto expression_actions_settings =
DB::ExpressionActionsSettings{getContext(), DB::CompileExpressions::yes};
auto function_capture =
std::make_shared<FunctionCaptureOverloadResolver>(
std::move(lambda_actions_dag),
expression_actions_settings,
diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRemove.cpp
b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRemove.cpp
index 430c3d1627..fdd76b271d 100644
--- a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRemove.cpp
+++ b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRemove.cpp
@@ -71,7 +71,7 @@ public:
NamesAndTypesList lambda_arguments_names_and_types;
lambda_arguments_names_and_types.emplace_back(x_in_lambda->result_name,
x_in_lambda->result_type);
DB::Names required_column_names =
lambda_actions_dag.getRequiredColumnsNames();
- auto expression_actions_settings =
DB::ExpressionActionsSettings::fromContext(getContext(),
DB::CompileExpressions::yes);
+ auto expression_actions_settings =
DB::ExpressionActionsSettings{getContext(), DB::CompileExpressions::yes};
auto function_capture =
std::make_shared<FunctionCaptureOverloadResolver>(
std::move(lambda_actions_dag),
expression_actions_settings,
diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRepeat.cpp
b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRepeat.cpp
index 348917f998..0c20296a2e 100644
--- a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRepeat.cpp
+++ b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRepeat.cpp
@@ -71,7 +71,7 @@ public:
NamesAndTypesList lambda_arguments_names_and_types;
lambda_arguments_names_and_types.emplace_back(x_in_lambda->result_name,
x_in_lambda->result_type);
DB::Names required_column_names =
lambda_actions_dag.getRequiredColumnsNames();
- auto expression_actions_settings =
DB::ExpressionActionsSettings::fromContext(getContext(),
DB::CompileExpressions::yes);
+ auto expression_actions_settings =
DB::ExpressionActionsSettings{getContext(), DB::CompileExpressions::yes};
auto function_capture =
std::make_shared<FunctionCaptureOverloadResolver>(
std::move(lambda_actions_dag),
expression_actions_settings,
diff --git
a/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.cpp
b/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.cpp
index 556474397b..79f3a3784a 100644
--- a/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.cpp
+++ b/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.cpp
@@ -130,7 +130,7 @@ public:
captured_column_names.push_back(required_column_name);
}
}
- auto expression_actions_settings =
DB::ExpressionActionsSettings::fromContext(getContext(),
DB::CompileExpressions::yes);
+ auto expression_actions_settings =
DB::ExpressionActionsSettings{getContext(), DB::CompileExpressions::yes};
auto function_capture =
std::make_shared<DB::FunctionCaptureOverloadResolver>(
std::move(lambda_actions_dag),
expression_actions_settings,
diff --git a/cpp-ch/local-engine/Storages/Cache/CacheManager.cpp
b/cpp-ch/local-engine/Storages/Cache/CacheManager.cpp
index f30e78dc91..f6421d1b2d 100644
--- a/cpp-ch/local-engine/Storages/Cache/CacheManager.cpp
+++ b/cpp-ch/local-engine/Storages/Cache/CacheManager.cpp
@@ -132,7 +132,9 @@ Task CacheManager::cachePart(
1);
QueryPlan plan;
plan.addStep(std::move(read_step));
- auto pipeline_builder = plan.buildQueryPipeline({}, {});
+ DB::QueryPlanOptimizationSettings optimization_settings{context};
+ DB::BuildQueryPipelineSettings build_settings{context};
+ auto pipeline_builder =
plan.buildQueryPipeline(optimization_settings, build_settings);
auto pipeline =
QueryPipelineBuilder::getPipeline(std::move(*pipeline_builder.get()));
PullingPipelineExecutor executor(pipeline);
while (true)
diff --git a/cpp-ch/local-engine/tests/gtest_ch_functions.cpp
b/cpp-ch/local-engine/tests/gtest_ch_functions.cpp
index c9bff195c5..fc094515a6 100644
--- a/cpp-ch/local-engine/tests/gtest_ch_functions.cpp
+++ b/cpp-ch/local-engine/tests/gtest_ch_functions.cpp
@@ -80,7 +80,7 @@ TEST(TestFunction, In)
set->insertFromBlock(col1_set_block.getColumnsWithTypeAndName());
set->finishInsert();
PreparedSets::Hash empty;
- auto future_set = std::make_shared<FutureSetFromStorage>(empty,
std::move(set));
+ auto future_set = std::make_shared<FutureSetFromStorage>(empty,
std::move(set), std::nullopt);
//TODO: WHY? after https://github.com/ClickHouse/ClickHouse/pull/63723 we
need pass 4 instead of 1
auto arg = ColumnSet::create(4, future_set);
@@ -124,7 +124,7 @@ TEST(TestFunction, NotIn1)
set->insertFromBlock(col1_set_block.getColumnsWithTypeAndName());
set->finishInsert();
PreparedSets::Hash empty;
- auto future_set = std::make_shared<FutureSetFromStorage>(empty,
std::move(set));
+ auto future_set = std::make_shared<FutureSetFromStorage>(empty,
std::move(set), std::nullopt);
//TODO: WHY? after https://github.com/ClickHouse/ClickHouse/pull/63723 we
need pass 4 instead of 1
auto arg = ColumnSet::create(4, future_set);
@@ -168,7 +168,7 @@ TEST(TestFunction, NotIn2)
set->insertFromBlock(col1_set_block.getColumnsWithTypeAndName());
set->finishInsert();
PreparedSets::Hash empty;
- auto future_set = std::make_shared<FutureSetFromStorage>(empty,
std::move(set));
+ auto future_set = std::make_shared<FutureSetFromStorage>(empty,
std::move(set), std::nullopt);
//TODO: WHY? after https://github.com/ClickHouse/ClickHouse/pull/63723 we
need pass 4 instead of 1
auto arg = ColumnSet::create(4, future_set);
diff --git a/cpp-ch/local-engine/tests/gtest_ch_join.cpp
b/cpp-ch/local-engine/tests/gtest_ch_join.cpp
index 5df5eaff8c..045d8d33c1 100644
--- a/cpp-ch/local-engine/tests/gtest_ch_join.cpp
+++ b/cpp-ch/local-engine/tests/gtest_ch_join.cpp
@@ -126,8 +126,8 @@ TEST(TestJoin, simple)
}
auto hash_join = std::make_shared<HashJoin>(join,
right_plan.getCurrentHeader());
- QueryPlanStepPtr join_step
- = std::make_unique<JoinStep>(left_plan.getCurrentHeader(),
right_plan.getCurrentHeader(), hash_join, 8192, 8192, 1, NameSet{}, false,
false);
+ QueryPlanStepPtr join_step = std::make_unique<JoinStep>(
+ left_plan.getCurrentHeader(), right_plan.getCurrentHeader(),
hash_join, 8192, 8192, 1, NameSet{}, false, false);
std::cerr << "join step:" << join_step->getOutputHeader().dumpStructure()
<< std::endl;
@@ -143,7 +143,8 @@ TEST(TestJoin, simple)
{NameWithAlias("colA", "colA"), NameWithAlias("colB", "colB"),
NameWithAlias("colD", "colD"), NameWithAlias("colC", "colC")});
QueryPlanStepPtr project_step =
std::make_unique<ExpressionStep>(query_plan.getCurrentHeader(),
std::move(project));
query_plan.addStep(std::move(project_step));
- auto pipeline =
query_plan.buildQueryPipeline(QueryPlanOptimizationSettings(),
BuildQueryPipelineSettings());
+ auto pipeline
+ =
query_plan.buildQueryPipeline(QueryPlanOptimizationSettings{global_context},
BuildQueryPipelineSettings{global_context});
auto executable_pipe =
QueryPipelineBuilder::getPipeline(std::move(*pipeline));
PullingPipelineExecutor executor(executable_pipe);
auto res = pipeline->getHeader().cloneEmpty();
diff --git a/cpp-ch/local-engine/tests/json/gtest_local_engine_config.json
b/cpp-ch/local-engine/tests/json/gtest_local_engine_config.json
index ea258487ff..5eaa473972 100644
--- a/cpp-ch/local-engine/tests/json/gtest_local_engine_config.json
+++ b/cpp-ch/local-engine/tests/json/gtest_local_engine_config.json
@@ -19,7 +19,6 @@
"spark.hadoop.input.read.timeout": "180000",
"spark.hadoop.fs.s3a.secret.key": "",
"spark.gluten.sql.columnar.backend.ch.runtime_config.hdfs.dfs_client_log_severity":
"INFO",
-
"spark.gluten.sql.columnar.backend.ch.runtime_settings.query_plan_enable_optimizations":
"false",
"spark.gluten.sql.columnar.backend.ch.runtime_config.storage_configuration.disks.hdfs_cache.disk":
"hdfs",
"spark.gluten.sql.columnar.backend.ch.runtime_config.storage_configuration.policies.__hdfs_main.volumes":
"main",
"spark.gluten.sql.columnar.backend.ch.runtime_settings.max_bytes_before_external_sort":
"5368709120",
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
