github-actions[bot] commented on code in PR #67397:
URL: https://github.com/apache/doris/pull/67397#discussion_r3943399744
##########
be/src/exprs/function/array/function_array_pop.cpp:
##########
@@ -104,9 +109,110 @@ class FunctionArrayPopfront : public
FunctionArrayPop<FunctionArrayPopfront> {
static constexpr int start_offset = 2;
};
+class FunctionArrayTrim : public IFunction {
+public:
+ static constexpr auto name = "trim_array";
+ static FunctionPtr create() { return
std::make_shared<FunctionArrayTrim>(); }
+
+ String get_name() const override { return name; }
+
+ bool is_variadic() const override { return false; }
+
+ size_t get_number_of_arguments() const override { return 2; }
+
+ bool use_default_implementation_for_nulls() const override { return false;
}
+
+ DataTypePtr get_return_type_impl(const DataTypes& arguments) const
override {
+ const auto array_type = remove_nullable(arguments[0]);
+ DCHECK(array_type->get_primitive_type() == TYPE_ARRAY)
+ << "First argument for function: " << name
+ << " should be DataTypeArray but it has type " <<
arguments[0]->get_name() << ".";
+ DCHECK(remove_nullable(arguments[1])->get_primitive_type() ==
TYPE_BIGINT)
+ << "Second argument for function: " << name << " should be
BigInt but it has type "
+ << arguments[1]->get_name() << ".";
+ if (arguments[0]->is_nullable() || arguments[1]->is_nullable()) {
+ return make_nullable(array_type);
+ }
+ return array_type;
+ }
+
+ Status execute_impl(FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
+ uint32_t result, size_t input_rows_count) const
override {
+ const auto& [array_column, array_is_const] =
+ unpack_if_const(block.get_by_position(arguments[0]).column);
+ const auto& [size_column, size_is_const] =
+ unpack_if_const(block.get_by_position(arguments[1]).column);
+
+ ColumnArrayExecutionData src;
+ if (!extract_column_array_info(*array_column, src)) {
+ return Status::RuntimeError(
+ fmt::format("execute failed, unsupported types for
function {}({}, {})",
+ get_name(),
block.get_by_position(arguments[0]).type->get_name(),
+
block.get_by_position(arguments[1]).type->get_name()));
+ }
+
+ const UInt8* size_null_map = nullptr;
+ const IColumn* size_data_column = size_column.get();
+ if (const auto* nullable_size =
check_and_get_column<ColumnNullable>(size_data_column)) {
+ size_null_map = nullable_size->get_null_map_data().data();
+ size_data_column = &nullable_size->get_nested_column();
+ }
+ const auto& sizes = assert_cast<const
ColumnInt64&>(*size_data_column).get_data();
+
+ auto result_array =
ColumnArray::create(src.array_col->get_data_ptr()->clone_empty(),
+
ColumnArray::ColumnOffsets::create());
+ auto& result_data = result_array->get_data();
+ auto& result_offsets = result_array->get_offsets();
+ result_data.reserve(src.array_col->get_data().size());
Review Comment:
`result_data.reserve` uses the physical source element count before any trim
size is examined. For a block that trims large numeric/nullable arrays
completely, this eagerly allocates almost another full input buffer (and
nullable null map) while the input is still live, even though the result data
is empty; near the memory limit that can fail an otherwise tiny-result query. A
const array has the opposite problem because its physical source contains one
array while the logical result may repeat it for every row, so this estimate
under-reserves and grows repeatedly. This is separate from the earlier
per-element insertion thread. Please validate sizes/build offsets in a first
pass, compute the overflow-checked total retained element count, reserve that
exact total, and then bulk-copy the retained ranges.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
