This is an automated email from the ASF dual-hosted git repository.
changchen pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new bae4dbcbed [GLUTEN-1632][CH]Daily Update Clickhouse Version (20241205)
(#8152)
bae4dbcbed is described below
commit bae4dbcbed6cd257a8c7f329f7924d3ef84df3ee
Author: Kyligence Git <[email protected]>
AuthorDate: Thu Dec 5 03:44:48 2024 -0600
[GLUTEN-1632][CH]Daily Update Clickhouse Version (20241205) (#8152)
* [GLUTEN-1632][CH]Daily Update Clickhouse Version (20241205)
* Fix Build due to https://github.com/ClickHouse/ClickHouse/pull/72747
* make LiteralParser::parse static
---------
Co-authored-by: kyligence-git <[email protected]>
Co-authored-by: Chang Chen <[email protected]>
---
cpp-ch/clickhouse.version | 4 +-
cpp-ch/local-engine/Common/AggregateUtil.cpp | 22 ++++++---
.../Parser/AggregateFunctionParser.cpp | 2 +-
cpp-ch/local-engine/Parser/ExpressionParser.cpp | 32 +++++-------
cpp-ch/local-engine/Parser/ExpressionParser.h | 12 ++---
cpp-ch/local-engine/Parser/FunctionParser.cpp | 2 +-
.../Parser/scalar_function_parser/arrayExcept.cpp | 27 +++++-----
.../arrayHighOrderFunctions.cpp | 4 +-
.../Parser/scalar_function_parser/arrayRemove.cpp | 10 ++--
.../Parser/scalar_function_parser/arrayRepeat.cpp | 10 ++--
.../scalar_function_parser/lambdaFunction.cpp | 57 +++++++++-------------
cpp-ch/local-engine/tests/gtest_ch_functions.cpp | 9 ++--
12 files changed, 90 insertions(+), 101 deletions(-)
diff --git a/cpp-ch/clickhouse.version b/cpp-ch/clickhouse.version
index 999c8c9572..7c94e253c7 100644
--- a/cpp-ch/clickhouse.version
+++ b/cpp-ch/clickhouse.version
@@ -1,3 +1,3 @@
CH_ORG=Kyligence
-CH_BRANCH=rebase_ch/20241204
-CH_COMMIT=7b72d0dc991
+CH_BRANCH=rebase_ch/20241205
+CH_COMMIT=7b8b1c0b554
diff --git a/cpp-ch/local-engine/Common/AggregateUtil.cpp
b/cpp-ch/local-engine/Common/AggregateUtil.cpp
index a6fedcc535..36b26e94bd 100644
--- a/cpp-ch/local-engine/Common/AggregateUtil.cpp
+++ b/cpp-ch/local-engine/Common/AggregateUtil.cpp
@@ -237,19 +237,29 @@ DB::Aggregator::Params
AggregatorParamsHelper::buildParams(
aggregate_settings[DB::Setting::max_block_size] = max_block_size;
aggregate_settings[DB::Setting::enable_software_prefetch_in_aggregation] =
enable_prefetch;
aggregate_settings[DB::Setting::optimize_group_by_constant_keys] =
optimize_group_by_constant_keys;
- DB::Aggregator::Params params(
- aggregate_settings,
+ return DB::Aggregator::Params{
grouping_keys,
agg_descriptions,
- false,
+ /*overflow_row*/ false,
+ aggregate_settings[DB::Setting::max_rows_to_group_by],
+ aggregate_settings[DB::Setting::group_by_overflow_mode],
group_by_two_level_threshold,
group_by_two_level_threshold_bytes,
+ DB::Aggregator::Params::getMaxBytesBeforeExternalGroupBy(
+
aggregate_settings[DB::Setting::max_bytes_before_external_group_by],
+
aggregate_settings[DB::Setting::max_bytes_ratio_before_external_group_by]),
empty_result_for_aggregation_by_empty_set,
tmp_data_scope,
+ aggregate_settings[DB::Setting::max_threads],
+
aggregate_settings[DB::Setting::min_free_disk_space_for_temporary_data],
+ aggregate_settings[DB::Setting::compile_aggregate_expressions],
+
aggregate_settings[DB::Setting::min_count_to_compile_aggregate_expression],
+ aggregate_settings[DB::Setting::max_block_size],
+
aggregate_settings[DB::Setting::enable_software_prefetch_in_aggregation],
only_merge,
- {});
-
- return params;
+ aggregate_settings[DB::Setting::optimize_group_by_constant_keys],
+
aggregate_settings[DB::Setting::min_hit_rate_to_use_consecutive_keys_optimization],
+ {}};
}
diff --git a/cpp-ch/local-engine/Parser/AggregateFunctionParser.cpp
b/cpp-ch/local-engine/Parser/AggregateFunctionParser.cpp
index eb05b26dcf..5452972c6f 100644
--- a/cpp-ch/local-engine/Parser/AggregateFunctionParser.cpp
+++ b/cpp-ch/local-engine/Parser/AggregateFunctionParser.cpp
@@ -76,7 +76,7 @@ const DB::ActionsDAG::Node *
AggregateFunctionParser::parseExpression(DB::Action
std::pair<DataTypePtr, Field> AggregateFunctionParser::parseLiteral(const
substrait::Expression_Literal & literal) const
{
- return LiteralParser().parse(literal);
+ return LiteralParser::parse(literal);
}
DB::ActionsDAG::NodeRawConstPtrs
diff --git a/cpp-ch/local-engine/Parser/ExpressionParser.cpp
b/cpp-ch/local-engine/Parser/ExpressionParser.cpp
index 9899e90ddc..e7de46483e 100644
--- a/cpp-ch/local-engine/Parser/ExpressionParser.cpp
+++ b/cpp-ch/local-engine/Parser/ExpressionParser.cpp
@@ -15,7 +15,7 @@
* limitations under the License.
*/
#include "ExpressionParser.h"
-#include <DataTypes/DataTypeAggregateFunction.h>
+#include <Core/Settings.h>
#include <DataTypes/DataTypeArray.h>
#include <DataTypes/DataTypeDate32.h>
#include <DataTypes/DataTypeDateTime64.h>
@@ -31,17 +31,13 @@
#include <DataTypes/Serializations/ISerialization.h>
#include <DataTypes/getLeastSupertype.h>
#include <IO/WriteBufferFromString.h>
-#include <Parser/ExpressionParser.h>
#include <Parser/FunctionParser.h>
#include <Parser/ParserContext.h>
#include <Parser/SerializedPlanParser.h>
#include <Parser/TypeParser.h>
-#include <Poco/Logger.h>
#include <Common/BlockTypeUtils.h>
#include <Common/CHUtil.h>
-#include <Common/StringUtils.h>
#include <Common/logger_useful.h>
-#include "SerializedPlanParser.h"
namespace DB
{
@@ -55,7 +51,7 @@ extern const int BAD_ARGUMENTS;
namespace local_engine
{
-std::pair<DB::DataTypePtr, DB::Field> LiteralParser::parse(const
substrait::Expression_Literal & literal) const
+std::pair<DB::DataTypePtr, DB::Field> LiteralParser::parse(const
substrait::Expression_Literal & literal)
{
DB::DataTypePtr type;
DB::Field field;
@@ -274,7 +270,7 @@ const ActionsDAG::Node *
ExpressionParser::parseExpression(ActionsDAG & actions_
case substrait::Expression::RexTypeCase::kLiteral: {
DB::DataTypePtr type;
DB::Field field;
- std::tie(type, field) = LiteralParser().parse(rel.literal());
+ std::tie(type, field) = LiteralParser::parse(rel.literal());
return addConstColumn(actions_dag, type, field);
}
@@ -333,7 +329,8 @@ const ActionsDAG::Node *
ExpressionParser::parseExpression(ActionsDAG & actions_
// ISSUE-7389: spark cast(map to string) has different
behavior with CH cast(map to string)
auto map_input_type = std::static_pointer_cast<const
DataTypeMap>(denull_input_type);
args.emplace_back(addConstColumn(actions_dag,
map_input_type->getKeyType(), map_input_type->getKeyType()->getDefault()));
- args.emplace_back(addConstColumn(actions_dag,
map_input_type->getValueType(), map_input_type->getValueType()->getDefault()));
+ args.emplace_back(
+ addConstColumn(actions_dag,
map_input_type->getValueType(), map_input_type->getValueType()->getDefault()));
result_node = toFunctionNode(actions_dag,
"sparkCastMapToString", args);
}
else if (isString(denull_input_type) && substrait_type.has_bool_())
@@ -411,7 +408,7 @@ const ActionsDAG::Node *
ExpressionParser::parseExpression(ActionsDAG & actions_
args.emplace_back(parseExpression(actions_dag,
rel.singular_or_list().value()));
bool nullable = false;
- int options_len = static_cast<int>(options.size());
+ int options_len = options.size();
for (int i = 0; i < options_len; ++i)
{
if (!options[i].has_literal())
@@ -421,15 +418,14 @@ const ActionsDAG::Node *
ExpressionParser::parseExpression(ActionsDAG & actions_
}
DB::DataTypePtr elem_type;
- LiteralParser literal_parser;
- std::tie(elem_type, std::ignore) =
literal_parser.parse(options[0].literal());
+ std::tie(elem_type, std::ignore) =
LiteralParser::parse(options[0].literal());
elem_type = wrapNullableType(nullable, elem_type);
DB::MutableColumnPtr elem_column = elem_type->createColumn();
elem_column->reserve(options_len);
for (int i = 0; i < options_len; ++i)
{
- auto type_and_field =
LiteralParser().parse(options[i].literal());
+ auto type_and_field =
LiteralParser::parse(options[i].literal());
auto option_type = wrapNullableType(nullable,
type_and_field.first);
if (!elem_type->equals(*option_type))
throw DB::Exception(
@@ -440,16 +436,12 @@ const ActionsDAG::Node *
ExpressionParser::parseExpression(ActionsDAG & actions_
elem_column->insert(type_and_field.second);
}
-
- DB::MutableColumns elem_columns;
- elem_columns.emplace_back(std::move(elem_column));
-
auto name = getUniqueName("__set");
- DB::Block elem_block;
- elem_block.insert(DB::ColumnWithTypeAndName(nullptr, elem_type,
name));
- elem_block.setColumns(std::move(elem_columns));
+ ColumnWithTypeAndName elem_block{std::move(elem_column),
elem_type, name};
- auto future_set =
std::make_shared<DB::FutureSetFromTuple>(elem_block,
context->queryContext()->getSettingsRef());
+ PreparedSets prepared_sets;
+ FutureSet::Hash emptyKey;
+ auto future_set = prepared_sets.addFromTuple(emptyKey,
{elem_block}, context->queryContext()->getSettingsRef());
auto arg = DB::ColumnSet::create(1, std::move(future_set));
args.emplace_back(&actions_dag.addColumn(DB::ColumnWithTypeAndName(std::move(arg),
std::make_shared<DB::DataTypeSet>(), name)));
diff --git a/cpp-ch/local-engine/Parser/ExpressionParser.h
b/cpp-ch/local-engine/Parser/ExpressionParser.h
index 95c66146ab..06a80d756e 100644
--- a/cpp-ch/local-engine/Parser/ExpressionParser.h
+++ b/cpp-ch/local-engine/Parser/ExpressionParser.h
@@ -16,17 +16,16 @@
*/
#pragma once
+#include <atomic>
#include <DataTypes/IDataType.h>
#include <Interpreters/ActionsDAG.h>
#include <Interpreters/Context_fwd.h>
#include <substrait/plan.pb.h>
-#include <atomic>
-#include "SerializedPlanParser.h"
namespace local_engine
{
-struct ParserContext;
+class ParserContext;
class SerializedPlanParser;
class LiteralParser
@@ -34,17 +33,17 @@ class LiteralParser
public:
/// Parse a substrait literal into a CH field
/// returns are the type and field value.
- std::pair<DB::DataTypePtr, DB::Field> parse(const
substrait::Expression_Literal & literal) const;
+ static std::pair<DB::DataTypePtr, DB::Field> parse(const
substrait::Expression_Literal & literal);
};
class ExpressionParser
{
public:
- ExpressionParser(std::shared_ptr<const ParserContext> context_) :
context(context_) { }
+ ExpressionParser(const std::shared_ptr<const ParserContext> & context_) :
context(context_) { }
~ExpressionParser() = default;
- /// Append a counter suffix to name
+ /// Append a counter-suffix to name
String getUniqueName(const String & name) const;
const DB::ActionsDAG::Node * addConstColumn(DB::ActionsDAG & actions_dag,
const DB::DataTypePtr type, const DB::Field & field) const;
@@ -84,6 +83,5 @@ private:
const substrait::Expression_ScalarFunction & func, DB::ActionsDAG &
actions_dag, bool position, bool & is_map) const;
DB::ActionsDAG::NodeRawConstPtrs parseJsonTuple(const
substrait::Expression_ScalarFunction & func, DB::ActionsDAG & actions_dag)
const;
-
};
}
diff --git a/cpp-ch/local-engine/Parser/FunctionParser.cpp
b/cpp-ch/local-engine/Parser/FunctionParser.cpp
index 7e794dabec..581ab65f61 100644
--- a/cpp-ch/local-engine/Parser/FunctionParser.cpp
+++ b/cpp-ch/local-engine/Parser/FunctionParser.cpp
@@ -92,7 +92,7 @@ const DB::ActionsDAG::Node *
FunctionParser::parseExpression(DB::ActionsDAG & ac
std::pair<DataTypePtr, Field> FunctionParser::parseLiteral(const
substrait::Expression_Literal & literal) const
{
- return LiteralParser().parse(literal);
+ return LiteralParser::parse(literal);
}
ActionsDAG::NodeRawConstPtrs
diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayExcept.cpp
b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayExcept.cpp
index a9a0f305a0..935aad6244 100644
--- a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayExcept.cpp
+++ b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayExcept.cpp
@@ -69,16 +69,14 @@ public:
const auto * lambda_output = toFunctionNode(lambda_actions_dag, "not",
{has_in_lambda});
lambda_actions_dag.getOutputs().push_back(lambda_output);
lambda_actions_dag.removeUnusedActions(Names(1,
lambda_output->result_name));
-
- auto expression_actions_settings =
DB::ExpressionActionsSettings::fromContext(getContext(),
DB::CompileExpressions::yes);
- auto lambda_actions =
std::make_shared<DB::ExpressionActions>(std::move(lambda_actions_dag),
expression_actions_settings);
-
DB::Names captured_column_names{arr2_in_lambda->result_name};
NamesAndTypesList lambda_arguments_names_and_types;
lambda_arguments_names_and_types.emplace_back(x_in_lambda->result_name,
x_in_lambda->result_type);
- DB::Names required_column_names = lambda_actions->getRequiredColumns();
+ DB::Names required_column_names =
lambda_actions_dag.getRequiredColumnsNames();
+ auto expression_actions_settings =
DB::ExpressionActionsSettings::fromContext(getContext(),
DB::CompileExpressions::yes);
auto function_capture =
std::make_shared<FunctionCaptureOverloadResolver>(
- lambda_actions,
+ std::move(lambda_actions_dag),
+ expression_actions_settings,
captured_column_names,
lambda_arguments_names_and_types,
lambda_output->result_type,
@@ -97,13 +95,16 @@ public:
const auto * arr2_is_null_node = toFunctionNode(actions_dag, "isNull",
{arr2_arg});
const auto * null_array_node
= addColumnToActionsDAG(actions_dag,
std::make_shared<DataTypeNullable>(array_distinct_node->result_type), {});
- const auto * multi_if_node = toFunctionNode(actions_dag, "multiIf", {
- arr1_is_null_node,
- null_array_node,
- arr2_is_null_node,
- null_array_node,
- array_distinct_node,
- });
+ const auto * multi_if_node = toFunctionNode(
+ actions_dag,
+ "multiIf",
+ {
+ arr1_is_null_node,
+ null_array_node,
+ arr2_is_null_node,
+ null_array_node,
+ array_distinct_node,
+ });
return convertNodeTypeIfNeeded(substrait_func, multi_if_node,
actions_dag);
}
};
diff --git
a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayHighOrderFunctions.cpp
b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayHighOrderFunctions.cpp
index 7ab829585b..1d418ba073 100644
---
a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayHighOrderFunctions.cpp
+++
b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayHighOrderFunctions.cpp
@@ -218,14 +218,14 @@ private:
const auto var_expr =
expr.scalar_function().arguments()[0].value();
if (!var_expr.has_literal())
return false;
- auto [_, name] = LiteralParser().parse(var_expr.literal());
+ auto [_, name] = LiteralParser::parse(var_expr.literal());
return var == name.safeGet<String>();
};
auto is_int_value = [&](const substrait::Expression & expr, Int32 val)
{
if (!expr.has_literal())
return false;
- auto [_, x] = LiteralParser().parse(expr.literal());
+ auto [_, x] = LiteralParser::parse(expr.literal());
return val == x.safeGet<Int32>();
};
diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRemove.cpp
b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRemove.cpp
index 27bd9f84a9..9bc67111f0 100644
--- a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRemove.cpp
+++ b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRemove.cpp
@@ -66,16 +66,14 @@ public:
const auto * lambda_output = if_null_in_lambda;
lambda_actions_dag.getOutputs().push_back(lambda_output);
lambda_actions_dag.removeUnusedActions(Names(1,
lambda_output->result_name));
-
- auto expression_actions_settings =
DB::ExpressionActionsSettings::fromContext(getContext(),
DB::CompileExpressions::yes);
- auto lambda_actions =
std::make_shared<DB::ExpressionActions>(std::move(lambda_actions_dag),
expression_actions_settings);
-
DB::Names captured_column_names{elem_in_lambda->result_name};
NamesAndTypesList lambda_arguments_names_and_types;
lambda_arguments_names_and_types.emplace_back(x_in_lambda->result_name,
x_in_lambda->result_type);
- DB::Names required_column_names = lambda_actions->getRequiredColumns();
+ DB::Names required_column_names =
lambda_actions_dag.getRequiredColumnsNames();
+ auto expression_actions_settings =
DB::ExpressionActionsSettings::fromContext(getContext(),
DB::CompileExpressions::yes);
auto function_capture =
std::make_shared<FunctionCaptureOverloadResolver>(
- lambda_actions,
+ std::move(lambda_actions_dag),
+ expression_actions_settings,
captured_column_names,
lambda_arguments_names_and_types,
lambda_output->result_type,
diff --git a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRepeat.cpp
b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRepeat.cpp
index 04f4f64e7b..e7df0250c8 100644
--- a/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRepeat.cpp
+++ b/cpp-ch/local-engine/Parser/scalar_function_parser/arrayRepeat.cpp
@@ -66,16 +66,14 @@ public:
const auto * lambda_output = elem_in_lambda;
lambda_actions_dag.getOutputs().push_back(lambda_output);
lambda_actions_dag.removeUnusedActions(Names(1,
lambda_output->result_name));
-
- auto expression_actions_settings =
DB::ExpressionActionsSettings::fromContext(getContext(),
DB::CompileExpressions::yes);
- auto lambda_actions =
std::make_shared<DB::ExpressionActions>(std::move(lambda_actions_dag),
expression_actions_settings);
-
DB::Names captured_column_names{elem_in_lambda->result_name};
NamesAndTypesList lambda_arguments_names_and_types;
lambda_arguments_names_and_types.emplace_back(x_in_lambda->result_name,
x_in_lambda->result_type);
- DB::Names required_column_names = lambda_actions->getRequiredColumns();
+ DB::Names required_column_names =
lambda_actions_dag.getRequiredColumnsNames();
+ auto expression_actions_settings =
DB::ExpressionActionsSettings::fromContext(getContext(),
DB::CompileExpressions::yes);
auto function_capture =
std::make_shared<FunctionCaptureOverloadResolver>(
- lambda_actions,
+ std::move(lambda_actions_dag),
+ expression_actions_settings,
captured_column_names,
lambda_arguments_names_and_types,
lambda_output->result_type,
diff --git
a/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.cpp
b/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.cpp
index 2d61633e80..19e3bfca05 100644
--- a/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.cpp
+++ b/cpp-ch/local-engine/Parser/scalar_function_parser/lambdaFunction.cpp
@@ -17,10 +17,9 @@
#include <Functions/FunctionsMiscellaneous.h>
#include <Interpreters/ExpressionActions.h>
#include <Interpreters/ExpressionActionsSettings.h>
+#include <Parser/ExpressionParser.h>
#include <Parser/FunctionParser.h>
#include <Parser/TypeParser.h>
-#include <Parser/ExpressionParser.h>
-#include <Common/Exception.h>
#include <Poco/Logger.h>
#include <Common/Exception.h>
#include <Common/logger_useful.h>
@@ -29,7 +28,7 @@
namespace DB::ErrorCodes
{
- extern const int LOGICAL_ERROR;
+extern const int LOGICAL_ERROR;
}
namespace local_engine
@@ -44,12 +43,10 @@ DB::NamesAndTypesList
collectLambdaArguments(ParserContextPtr parser_context_, c
if (arg.value().has_scalar_function()
&&
parser_context_->getFunctionNameInSignature(arg.value().scalar_function().function_reference())
== "namedlambdavariable")
{
- auto [_, col_name_field] =
LiteralParser().parse(arg.value().scalar_function().arguments()[0].value().literal());
+ auto [_, col_name_field] =
LiteralParser::parse(arg.value().scalar_function().arguments()[0].value().literal());
String col_name = col_name_field.safeGet<String>();
if (collected_names.contains(col_name))
- {
continue;
- }
collected_names.insert(col_name);
auto type =
TypeParser::parseType(arg.value().scalar_function().output_type());
lambda_arguments.emplace_back(col_name, type);
@@ -63,7 +60,7 @@ class FunctionParserLambda : public FunctionParser
{
public:
static constexpr auto name = "lambdafunction";
- explicit FunctionParserLambda(ParserContextPtr parser_context_) :
FunctionParser(parser_context_) {}
+ explicit FunctionParserLambda(ParserContextPtr parser_context_) :
FunctionParser(parser_context_) { }
~FunctionParserLambda() override = default;
String getName() const override { return name; }
@@ -73,15 +70,14 @@ public:
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "getCHFunctionName
is not implemented for LambdaFunction");
}
- const DB::ActionsDAG::Node * parse(const
substrait::Expression_ScalarFunction & substrait_func, DB::ActionsDAG &
actions_dag) const override
+ const DB::ActionsDAG::Node *
+ parse(const substrait::Expression_ScalarFunction & substrait_func,
DB::ActionsDAG & actions_dag) const override
{
/// Some special cases, for example, `transform(arr, x -> concat(arr,
array(x)))` refers to
/// a column `arr` out of it directly. We need a `arr` as an input
column for `lambda_actions_dag`
DB::NamesAndTypesList parent_header;
for (const auto * output_node : actions_dag.getOutputs())
- {
parent_header.emplace_back(output_node->result_name,
output_node->result_type);
- }
ActionsDAG lambda_actions_dag{parent_header};
/// The first argument is the lambda function body, followings are the
lambda arguments which is
@@ -90,28 +86,21 @@ public:
/// this outside lambda function's arguments. For an example,
transform(number, x -> transform(letter, y -> struct(x, y))).
/// Before parsing the lambda function body, we add lambda function
arguments int actions dag at first.
for (size_t i = 1; i < substrait_func.arguments().size(); ++i)
- {
(void)parseExpression(lambda_actions_dag,
substrait_func.arguments()[i].value());
- }
const auto & substrait_lambda_body =
substrait_func.arguments()[0].value();
const auto * lambda_body_node = parseExpression(lambda_actions_dag,
substrait_lambda_body);
lambda_actions_dag.getOutputs().push_back(lambda_body_node);
lambda_actions_dag.removeUnusedActions(Names(1,
lambda_body_node->result_name));
- auto expression_actions_settings =
DB::ExpressionActionsSettings::fromContext(getContext(),
DB::CompileExpressions::yes);
- auto lambda_actions =
std::make_shared<DB::ExpressionActions>(std::move(lambda_actions_dag),
expression_actions_settings);
-
DB::Names captured_column_names;
- DB::Names required_column_names = lambda_actions->getRequiredColumns();
+ DB::Names required_column_names =
lambda_actions_dag.getRequiredColumnsNames();
DB::ActionsDAG::NodeRawConstPtrs lambda_children;
auto lambda_function_args = collectLambdaArguments(parser_context,
substrait_func);
- const auto & lambda_actions_inputs =
lambda_actions->getActionsDAG().getInputs();
+ const auto & lambda_actions_inputs = lambda_actions_dag.getInputs();
std::unordered_map<String, const DB::ActionsDAG::Node *> parent_nodes;
for (const auto & node : actions_dag.getNodes())
- {
parent_nodes[node.result_name] = &node;
- }
for (const auto & required_column_name : required_column_names)
{
if (std::find_if(
@@ -125,13 +114,13 @@ public:
lambda_actions_inputs.end(),
[&required_column_name](const auto & node) { return
node->result_name == required_column_name; });
if (it == lambda_actions_inputs.end())
- {
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR,
"Required column not found: {}", required_column_name);
- }
auto parent_node_it = parent_nodes.find(required_column_name);
if (parent_node_it == parent_nodes.end())
{
- throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Not
found column {} in actions dag:\n{}",
+ throw DB::Exception(
+ DB::ErrorCodes::LOGICAL_ERROR,
+ "Not found column {} in actions dag:\n{}",
required_column_name,
actions_dag.dumpDAG());
}
@@ -141,9 +130,10 @@ public:
captured_column_names.push_back(required_column_name);
}
}
-
+ auto expression_actions_settings =
DB::ExpressionActionsSettings::fromContext(getContext(),
DB::CompileExpressions::yes);
auto function_capture =
std::make_shared<DB::FunctionCaptureOverloadResolver>(
- lambda_actions,
+ std::move(lambda_actions_dag),
+ expression_actions_settings,
captured_column_names,
lambda_function_args,
lambda_body_node->result_type,
@@ -153,10 +143,10 @@ public:
const auto * result = &actions_dag.addFunction(function_capture,
lambda_children, lambda_body_node->result_name);
return result;
}
+
protected:
- DB::ActionsDAG::NodeRawConstPtrs parseFunctionArguments(
- const substrait::Expression_ScalarFunction & substrait_func,
- DB::ActionsDAG & actions_dag) const override
+ DB::ActionsDAG::NodeRawConstPtrs
+ parseFunctionArguments(const substrait::Expression_ScalarFunction &
substrait_func, DB::ActionsDAG & actions_dag) const override
{
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR,
"parseFunctionArguments is not implemented for LambdaFunction");
}
@@ -177,7 +167,7 @@ class NamedLambdaVariable : public FunctionParser
{
public:
static constexpr auto name = "namedlambdavariable";
- explicit NamedLambdaVariable(ParserContextPtr parser_context_) :
FunctionParser(parser_context_) {}
+ explicit NamedLambdaVariable(ParserContextPtr parser_context_) :
FunctionParser(parser_context_) { }
~NamedLambdaVariable() override = default;
String getName() const override { return name; }
@@ -187,7 +177,8 @@ public:
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "getCHFunctionName
is not implemented for NamedLambdaVariable");
}
- const DB::ActionsDAG::Node * parse(const
substrait::Expression_ScalarFunction & substrait_func, DB::ActionsDAG &
actions_dag) const override
+ const DB::ActionsDAG::Node *
+ parse(const substrait::Expression_ScalarFunction & substrait_func,
DB::ActionsDAG & actions_dag) const override
{
auto [_, col_name_field] =
parseLiteral(substrait_func.arguments()[0].value().literal());
String col_name = col_name_field.safeGet<String>();
@@ -196,15 +187,13 @@ public:
const auto & inputs = actions_dag.getInputs();
auto it = std::find_if(inputs.begin(), inputs.end(), [&col_name](const
auto * node) { return node->result_name == col_name; });
if (it == inputs.end())
- {
return &(actions_dag.addInput(col_name, type));
- }
return *it;
}
+
protected:
- DB::ActionsDAG::NodeRawConstPtrs parseFunctionArguments(
- const substrait::Expression_ScalarFunction & substrait_func,
- DB::ActionsDAG & actions_dag) const override
+ DB::ActionsDAG::NodeRawConstPtrs
+ parseFunctionArguments(const substrait::Expression_ScalarFunction &
substrait_func, DB::ActionsDAG & actions_dag) const override
{
throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR,
"parseFunctionArguments is not implemented for NamedLambdaVariable");
}
diff --git a/cpp-ch/local-engine/tests/gtest_ch_functions.cpp
b/cpp-ch/local-engine/tests/gtest_ch_functions.cpp
index 3b91e07994..c9bff195c5 100644
--- a/cpp-ch/local-engine/tests/gtest_ch_functions.cpp
+++ b/cpp-ch/local-engine/tests/gtest_ch_functions.cpp
@@ -79,7 +79,8 @@ TEST(TestFunction, In)
set->setHeader(col1_set_block.getColumnsWithTypeAndName());
set->insertFromBlock(col1_set_block.getColumnsWithTypeAndName());
set->finishInsert();
- auto future_set = std::make_shared<FutureSetFromStorage>(std::move(set));
+ PreparedSets::Hash empty;
+ auto future_set = std::make_shared<FutureSetFromStorage>(empty,
std::move(set));
//TODO: WHY? after https://github.com/ClickHouse/ClickHouse/pull/63723 we
need pass 4 instead of 1
auto arg = ColumnSet::create(4, future_set);
@@ -122,7 +123,8 @@ TEST(TestFunction, NotIn1)
set->setHeader(col1_set_block.getColumnsWithTypeAndName());
set->insertFromBlock(col1_set_block.getColumnsWithTypeAndName());
set->finishInsert();
- auto future_set = std::make_shared<FutureSetFromStorage>(std::move(set));
+ PreparedSets::Hash empty;
+ auto future_set = std::make_shared<FutureSetFromStorage>(empty,
std::move(set));
//TODO: WHY? after https://github.com/ClickHouse/ClickHouse/pull/63723 we
need pass 4 instead of 1
auto arg = ColumnSet::create(4, future_set);
@@ -165,7 +167,8 @@ TEST(TestFunction, NotIn2)
set->setHeader(col1_set_block.getColumnsWithTypeAndName());
set->insertFromBlock(col1_set_block.getColumnsWithTypeAndName());
set->finishInsert();
- auto future_set = std::make_shared<FutureSetFromStorage>(std::move(set));
+ PreparedSets::Hash empty;
+ auto future_set = std::make_shared<FutureSetFromStorage>(empty,
std::move(set));
//TODO: WHY? after https://github.com/ClickHouse/ClickHouse/pull/63723 we
need pass 4 instead of 1
auto arg = ColumnSet::create(4, future_set);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
