github-actions[bot] commented on code in PR #69040:
URL: https://github.com/apache/doris/pull/69040#discussion_r4262081677
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/types/DataType.java:
##########
@@ -916,6 +949,10 @@ private static void validateCatalogDataType(Type
catalogType) {
if (catalogType.isMapType()) {
org.apache.doris.catalog.MapType mt =
(org.apache.doris.catalog.MapType) catalogType;
+ if (mt.getKeyType().typeContainsFile()) {
Review Comment:
[P2] Apply the FILE key restriction to scalar MAP constructors
This check runs through `validateDataType`, but scalar `CreateMap` infers
its result without calling it and only rejects JSON/VARIANT keys. A valid
non-NULL FILE `f` therefore reaches this reduced plan:
```text
Project(element_at(map(f, 1, f, 2), f))
Project(CAST(named_struct('uri', concat('urn:', number),
'offset', NULL, 'size', NULL, 'content_type', NULL,
'checksum', NULL, 'inline', NULL) AS FILE) AS f)
Numbers
```
`FunctionMap` accepts these keys and `ColumnMap::deduplicate_keys`
deduplicates their serialized bytes, despite FILE having no SQL key semantics.
The subsequent lookup calls `FunctionArrayElement::_get_mapped_idx` and then
`ColumnFile::compare_at`, which throws `FILE comparison is not supported`.
Check each key argument with the recursive `Expression.checkFileKey` during
`CreateMap` binding so this forbidden map type cannot enter execution.
##########
be/src/exprs/function/cast/function_cast_file.cpp:
##########
@@ -0,0 +1,290 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <array>
+#include <cstring>
+
+#include "core/column/column_file.h"
+#include "core/column/column_struct.h"
+#include "core/data_type/data_type_file.h"
+#include "core/data_type/data_type_jsonb.h"
+#include "core/data_type/data_type_variant.h"
+#include "core/data_type/data_type_variant_v2.h"
+#include "core/data_type_serde/data_type_serde.h"
+#include "core/value/file_value.h"
+#include "exprs/function/cast/cast_to_variant.h"
+#include "exprs/function/cast/cast_wrapper_decls.h"
+#include "exprs/function/cast/variant_v2/cast_variant_v2.h"
+#include "util/jsonb_document.h"
+#include "util/jsonb_writer.h"
+
+namespace doris::CastWrapper {
+namespace {
+
+using FileFieldMapping = std::array<int, DataTypeFile::FIELD_COUNT>;
+
+bool compatible_file_field_type(size_t index, PrimitiveType primitive) {
+ if (index == 1 || index == 2) {
+ return primitive == TYPE_BIGINT;
+ }
+ if (index == DataTypeFile::INLINE_FIELD_INDEX) {
+ return primitive == TYPE_VARBINARY;
+ }
+ return is_string_type(primitive);
+}
+
+Status map_file_fields(const DataTypeStruct& source, FileFieldMapping&
mapping) {
+ if (source.get_elements().size() != DataTypeFile::FIELD_COUNT) {
+ return Status::InvalidArgument("FILE CAST requires exactly six
fields");
+ }
+ mapping.fill(-1);
+ const DataTypeFile file;
+ for (size_t i = 0; i < source.get_elements().size(); ++i) {
+ const auto& name = source.get_element_names()[i];
+ const auto position = file.try_get_position_by_name(name);
+ if (!position) {
+ return Status::InvalidArgument("Unknown FILE field {}", name);
+ }
+ if (mapping[*position] >= 0) {
+ return Status::InvalidArgument("Duplicate FILE field {}", name);
+ }
+ const auto type = remove_nullable(source.get_elements()[i]);
+ const auto primitive = type->get_primitive_type();
+ const bool safe = compatible_file_field_type(*position, primitive);
+ if (!safe && !type->is_null_literal()) {
+ return Status::InvalidArgument("Unsafe conversion of {} to FILE
field {}",
+ type->get_name(), name);
+ }
+ mapping[*position] = cast_set<int>(i);
+ }
+ return Status::OK();
+}
+
+Result<ColumnPtr> cast_struct_to_file(const ColumnStruct& structure,
+ const DataTypeStruct& structure_type,
+ const FileFieldMapping& mapping,
+ const NullMap::value_type* null_map) {
+ const auto file_type = std::make_shared<DataTypeFile>();
+ const auto rows = structure.size();
+ auto output = ColumnFile::create();
+ for (size_t i = 0; i < mapping.size(); ++i) {
+ const auto source_index = cast_set<size_t>(mapping[i]);
+ if (structure_type.get_elements()[source_index]->is_null_literal()) {
+ auto child = file_type->get_element(i)->create_column();
+ child->insert_many_defaults(rows);
+ output->get_column_ptr(i) = std::move(child);
+ } else {
+ output->get_column_ptr(i) = make_nullable(
+
structure.get_column_ptr(source_index)->convert_to_full_column_if_const());
+ }
+ }
+ const ColumnPtr prepared = std::move(output);
+ if (null_map) {
+ auto parent_nulls = ColumnUInt8::create(rows);
+ std::memcpy(parent_nulls->get_data().data(), null_map, rows);
+ const auto checked = ColumnNullable::create(prepared,
std::move(parent_nulls));
+ RETURN_IF_ERROR_RESULT(validate_file_column(*checked,
make_nullable(file_type)));
+ } else {
+ RETURN_IF_ERROR_RESULT(validate_file_column(*prepared, file_type));
+ }
+ return prepared;
+}
+
+WrapperType create_cast_to_file(const DataTypePtr& from_type) {
+ FileFieldMapping mapping {};
+ const bool from_json = from_type->get_primitive_type() == TYPE_JSONB;
+ if (!from_json) {
+ const auto* source =
check_and_get_data_type<DataTypeStruct>(from_type.get());
+ if (!source) return create_unsupport_wrapper(from_type->get_name(),
"FILE");
+ auto status = map_file_fields(*source, mapping);
+ if (!status.ok()) return
create_unsupport_wrapper(std::string(status.msg()));
+ }
+ return [from_type, from_json, mapping](FunctionContext* context, Block&
block,
+ const ColumnNumbers& arguments,
uint32_t result,
+ size_t rows, const
NullMap::value_type* null_map) {
+ const auto source =
+
block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();
+ if (!from_json) {
+ block.get_by_position(result).column =
DORIS_TRY(cast_struct_to_file(
+ assert_cast<const ColumnStruct&>(*source),
+ assert_cast<const DataTypeStruct&>(*from_type), mapping,
null_map));
+ return Status::OK();
+ }
+ const auto file_type = std::make_shared<DataTypeFile>();
+ const auto serde = file_type->get_serde();
+ auto output = file_type->create_column();
+ auto output_nulls = ColumnUInt8::create(rows, 0);
+ auto& nulls = output_nulls->get_data();
+ bool has_result_null = false;
+ CastParameters parameters;
+ parameters.is_strict = context->enable_strict_mode();
+ for (size_t row = 0; row < rows; ++row) {
+ if (null_map && null_map[row]) {
+ // The enclosing nullable wrapper owns this NULL; the FILE is a
+ // placeholder.
+ output->insert_default();
+ continue;
+ }
+ Status status;
+ const auto bytes = source->get_data_at(row);
+ const JsonbDocument* document = nullptr;
+ status = JsonbDocument::checkAndCreateDocument(bytes.data,
bytes.size, &document);
+ if (status.ok()) {
+ const auto* value = document->getValue();
+ if (value->isNull()) {
+ output->insert_default();
+ nulls[row] = 1;
+ has_result_null = true;
+ continue;
+ }
+ status = serde->deserialize_column_from_jsonb(*output, value,
parameters);
+ }
+ RETURN_IF_ERROR(status);
+ }
+ if (has_result_null) {
+ block.get_by_position(result).column =
+ ColumnNullable::create(std::move(output),
std::move(output_nulls));
+ } else {
+ block.get_by_position(result).column = std::move(output);
+ }
+ return Status::OK();
+ };
+}
+
+WrapperType create_cast_from_file(const DataTypePtr& from_type, const
DataTypePtr& to_type) {
+ const auto& file_type = assert_cast<const DataTypeFile&>(*from_type);
+ const auto primitive = to_type->get_primitive_type();
+ if (primitive == TYPE_STRUCT) {
+ const auto& structure = assert_cast<const DataTypeStruct&>(*to_type);
+ FileFieldMapping mapping {};
+ auto status = map_file_fields(structure, mapping);
+ if (!status.ok()) {
+ return create_unsupport_wrapper(std::string(status.msg()));
+ }
+ DataTypes fields;
+ FileFieldMapping source_positions {};
+ for (size_t file_index = 0; file_index < mapping.size(); ++file_index)
{
+ source_positions[mapping[file_index]] = cast_set<int>(file_index);
+ }
+ for (size_t target_index = 0; target_index < source_positions.size();
++target_index) {
+ const auto& field = structure.get_elements()[target_index];
+ const auto file_index =
cast_set<size_t>(source_positions[target_index]);
+ const auto field_primitive =
remove_nullable(field)->get_primitive_type();
+ const bool compatible = compatible_file_field_type(file_index,
field_primitive);
+ if (!field->is_nullable() || !compatible) {
+ return create_unsupport_wrapper("FILE can only cast to its
six-field STRUCT");
+ }
+ fields.push_back(file_type.get_elements()[file_index]);
+ }
+ const auto structure_type =
+ std::make_shared<DataTypeStruct>(fields,
structure.get_element_names());
+ return [structure_type, to_type, source_positions](
+ FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
+ uint32_t result, size_t rows, const
NullMap::value_type* null_map) {
+ const auto source =
+
block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();
+ const auto& file = assert_cast<const ColumnFile&>(*source);
+ Columns children;
+ for (size_t i = 0; i < DataTypeFile::FIELD_COUNT; ++i) {
+ children.push_back(file.get_column_ptr(source_positions[i]));
+ }
+ Block projected {{ColumnStruct::create(children), structure_type,
"file_fields"},
+ {nullptr, to_type, "result"}};
+ RETURN_IF_ERROR(prepare_unpack_dictionaries(context,
structure_type, to_type)(
+ context, projected, {0}, 1, rows, null_map));
+ block.get_by_position(result).column =
projected.get_by_position(1).column;
+ return Status::OK();
+ };
+ }
+ if (primitive != TYPE_JSONB) {
+ return create_unsupport_wrapper(from_type->get_name(),
to_type->get_name());
+ }
+ return [from_type](FunctionContext*, Block& block, const ColumnNumbers&
arguments,
+ uint32_t result, size_t rows, const
NullMap::value_type* null_map) {
+ const auto& source = block.get_by_position(arguments[0]).column;
+ const auto serde = from_type->get_serde();
+ auto output = ColumnString::create();
+ for (size_t row = 0; row < rows; ++row) {
+ if (null_map && null_map[row]) {
+ output->insert_default();
+ continue;
+ }
+ JsonbWriter writer;
+ RETURN_IF_ERROR(serde->serialize_column_to_jsonb(*source, row,
writer));
+ output->insert_data(writer.getOutput()->getBuffer(),
writer.getOutput()->getSize());
+ }
+ block.get_by_position(result).column = std::move(output);
+ return Status::OK();
+ };
+}
+
+} // namespace
+
+WrapperType create_file_wrapper(const DataTypePtr& from_type, const
DataTypePtr& to_type) {
+ if (from_type->get_primitive_type() == TYPE_VARIANT) {
+ return [from_type](FunctionContext* context, Block& block, const
ColumnNumbers& arguments,
+ uint32_t result, size_t rows, const
NullMap::value_type* null_map) {
+ const auto json_type = std::make_shared<DataTypeJsonb>();
+ const auto file_type = std::make_shared<DataTypeFile>();
+ Block intermediate {
+
{block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(),
+ from_type, "variant"},
+ {nullptr, make_nullable(json_type), "json"},
+ {nullptr, make_nullable(file_type), "file"}};
+ const auto* legacy_variant = dynamic_cast<const
DataTypeVariant*>(from_type.get());
+ auto to_json = legacy_variant
+ ?
create_cast_from_variant_wrapper(*legacy_variant, json_type)
+ :
create_cast_from_variant_v2_wrapper(json_type);
+ RETURN_IF_ERROR(to_json(context, intermediate, {0}, 1, rows,
null_map));
+ RETURN_IF_ERROR(prepare_unpack_dictionaries(context,
make_nullable(json_type),
+
make_nullable(file_type))(
+ context, intermediate, {1}, 2, rows, nullptr));
Review Comment:
[P2] Preserve the parent NULL mask in the JSONB-to-FILE step
A mixed batch can reach this reduced plan with a stored legacy VARIANT
column:
```text
Project(CAST(IF(id=1, NULL, named_struct('v', v)) AS STRUCT<v:FILE>))
Scan(id, v VARIANT)
```
Let row 1 contain a non-FILE object such as {"unrelated":1}, and row 2 a
valid six-field FILE object. `execute_for_null_then_else` keeps row 1's nested
payload beneath its NULL STRUCT, and the struct/nullable cast wrappers pass
that parent mask here. The legacy `cast_from_variant_impl` object-to-JSONB path
ignores the mask, however, and this second conversion receives `nullptr`; FILE
deserialization therefore validates the hidden object and fails the whole query
instead of returning NULL for row 1. Forward `null_map` into this conversion,
or merge it into the intermediate JSONB null map before validation.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
