This is an automated email from the ASF dual-hosted git repository. HappenLee pushed a commit to branch paimon-blob-4.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 6b748317f7d4f8c1966ecf6513c64b9a65b5df5f Author: linzhenqi <[email protected]> AuthorDate: Thu Oct 22 01:54:42 2026 +0800 [fix](type) Restrict FILE casts to STRUCT and support TO_JSON Issue Number: N/A Problem Summary: Allow explicit FILE conversion only with its six-field STRUCT representation, while preserving identity and NULL handling. Reject JSON and VARIANT casts, including nested FILE values, and accept FILE arguments in TO_JSON. FILE supports TO_JSON and explicit conversion to and from its six-field STRUCT. - Test: Full BE/FE build; 42 FE unit tests; 32 BE unit tests; 7 regression suites. - Behavior changed: Yes - Does this need documentation: Yes, updated in doris-website. (cherry picked from commit d574129047aeee2a41b2f7b9bb2b9c5c94af9ad8) --- be/src/exprs/function/cast/function_cast.cpp | 4 +- be/src/exprs/function/cast/function_cast_file.cpp | 143 ++--------- be/test/exprs/file_expr_test.cpp | 275 +++++++-------------- .../nereids/rules/expression/check/CheckCast.java | 6 +- .../trees/expressions/functions/scalar/ToJson.java | 2 +- .../nereids/types/FileTypeExpressionTest.java | 44 +++- .../datatype_p0/complex_types/test_file_type.out | 20 +- .../complex_types/test_file_type.groovy | 104 ++++---- .../complex_types/test_file_type_optimizer.groovy | 4 +- .../query_p0/cache/test_file_type_sql_cache.groovy | 16 +- 10 files changed, 222 insertions(+), 396 deletions(-) diff --git a/be/src/exprs/function/cast/function_cast.cpp b/be/src/exprs/function/cast/function_cast.cpp index a922f8b8f2a..e6881caa7c3 100644 --- a/be/src/exprs/function/cast/function_cast.cpp +++ b/be/src/exprs/function/cast/function_cast.cpp @@ -253,7 +253,9 @@ WrapperType prepare_impl(FunctionContext* context, const DataTypePtr& origin_fro return create_file_wrapper(from_type, to_type); } if ((contains_file_type(from_type) || contains_file_type(to_type)) && - (from_type->get_primitive_type() == TYPE_VARIANT || + (from_type->get_primitive_type() == TYPE_JSONB || + to_type->get_primitive_type() == TYPE_JSONB || + from_type->get_primitive_type() == TYPE_VARIANT || to_type->get_primitive_type() == TYPE_VARIANT || is_string_type(from_type->get_primitive_type()))) { return create_unsupport_wrapper(from_type->get_name(), to_type->get_name()); diff --git a/be/src/exprs/function/cast/function_cast_file.cpp b/be/src/exprs/function/cast/function_cast_file.cpp index 4b82225e2aa..636a2b7991c 100644 --- a/be/src/exprs/function/cast/function_cast_file.cpp +++ b/be/src/exprs/function/cast/function_cast_file.cpp @@ -21,16 +21,10 @@ #include "core/column/column_file.h" #include "core/column/column_struct.h" #include "core/data_type/data_type_file.h" -#include "core/data_type/data_type_jsonb.h" -#include "core/data_type/data_type_variant.h" -#include "core/data_type/data_type_variant_v2.h" -#include "core/data_type_serde/data_type_serde.h" +#include "core/data_type/data_type_struct.h" #include "core/value/file_value.h" #include "exprs/function/cast/cast_to_variant.h" #include "exprs/function/cast/cast_wrapper_decls.h" -#include "exprs/function/cast/variant_v2/cast_variant_v2.h" -#include "util/jsonb_document.h" -#include "util/jsonb_writer.h" namespace doris::CastWrapper { namespace { @@ -106,61 +100,21 @@ Result<ColumnPtr> cast_struct_to_file(const ColumnStruct& structure, WrapperType create_cast_to_file(const DataTypePtr& from_type) { FileFieldMapping mapping {}; - const bool from_json = from_type->get_primitive_type() == TYPE_JSONB; - if (!from_json) { - const auto* source = check_and_get_data_type<DataTypeStruct>(from_type.get()); - if (!source) return create_unsupport_wrapper(from_type->get_name(), "FILE"); - auto status = map_file_fields(*source, mapping); - if (!status.ok()) return create_unsupport_wrapper(std::string(status.msg())); + const auto* source = check_and_get_data_type<DataTypeStruct>(from_type.get()); + if (!source) { + return create_unsupport_wrapper(from_type->get_name(), "FILE"); } - return [from_type, from_json, mapping](FunctionContext* context, Block& block, - const ColumnNumbers& arguments, uint32_t result, - size_t rows, const NullMap::value_type* null_map) { + auto status = map_file_fields(*source, mapping); + if (!status.ok()) { + return create_unsupport_wrapper(std::string(status.msg())); + } + return [from_type, mapping](FunctionContext*, Block& block, const ColumnNumbers& arguments, + uint32_t result, size_t, const NullMap::value_type* null_map) { const auto source = block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(); - if (!from_json) { - block.get_by_position(result).column = DORIS_TRY(cast_struct_to_file( - assert_cast<const ColumnStruct&>(*source), - assert_cast<const DataTypeStruct&>(*from_type), mapping, null_map)); - return Status::OK(); - } - const auto file_type = std::make_shared<DataTypeFile>(); - const auto serde = file_type->get_serde(); - auto output = file_type->create_column(); - auto output_nulls = ColumnUInt8::create(rows, 0); - auto& nulls = output_nulls->get_data(); - bool has_result_null = false; - CastParameters parameters; - parameters.is_strict = context->enable_strict_mode(); - for (size_t row = 0; row < rows; ++row) { - if (null_map && null_map[row]) { - // The enclosing nullable wrapper owns this NULL; the FILE is a - // placeholder. - output->insert_default(); - continue; - } - Status status; - const auto bytes = source->get_data_at(row); - const JsonbDocument* document = nullptr; - status = JsonbDocument::checkAndCreateDocument(bytes.data, bytes.size, &document); - if (status.ok()) { - const auto* value = document->getValue(); - if (value->isNull()) { - output->insert_default(); - nulls[row] = 1; - has_result_null = true; - continue; - } - status = serde->deserialize_column_from_jsonb(*output, value, parameters); - } - RETURN_IF_ERROR(status); - } - if (has_result_null) { - block.get_by_position(result).column = - ColumnNullable::create(std::move(output), std::move(output_nulls)); - } else { - block.get_by_position(result).column = std::move(output); - } + block.get_by_position(result).column = DORIS_TRY(cast_struct_to_file( + assert_cast<const ColumnStruct&>(*source), + assert_cast<const DataTypeStruct&>(*from_type), mapping, null_map)); return Status::OK(); }; } @@ -210,80 +164,15 @@ WrapperType create_cast_from_file(const DataTypePtr& from_type, const DataTypePt return Status::OK(); }; } - if (primitive != TYPE_JSONB) { - return create_unsupport_wrapper(from_type->get_name(), to_type->get_name()); - } - return [from_type](FunctionContext*, Block& block, const ColumnNumbers& arguments, - uint32_t result, size_t rows, const NullMap::value_type* null_map) { - const auto& source = block.get_by_position(arguments[0]).column; - const auto serde = from_type->get_serde(); - auto output = ColumnString::create(); - for (size_t row = 0; row < rows; ++row) { - if (null_map && null_map[row]) { - output->insert_default(); - continue; - } - JsonbWriter writer; - RETURN_IF_ERROR(serde->serialize_column_to_jsonb(*source, row, writer)); - output->insert_data(writer.getOutput()->getBuffer(), writer.getOutput()->getSize()); - } - block.get_by_position(result).column = std::move(output); - return Status::OK(); - }; + return create_unsupport_wrapper(from_type->get_name(), to_type->get_name()); } } // namespace WrapperType create_file_wrapper(const DataTypePtr& from_type, const DataTypePtr& to_type) { - if (from_type->get_primitive_type() == TYPE_VARIANT) { - return [from_type](FunctionContext* context, Block& block, const ColumnNumbers& arguments, - uint32_t result, size_t rows, const NullMap::value_type* null_map) { - const auto json_type = std::make_shared<DataTypeJsonb>(); - const auto file_type = std::make_shared<DataTypeFile>(); - Block intermediate { - {block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(), - from_type, "variant"}, - {nullptr, make_nullable(json_type), "json"}, - {nullptr, make_nullable(file_type), "file"}}; - const auto* legacy_variant = dynamic_cast<const DataTypeVariant*>(from_type.get()); - auto to_json = legacy_variant - ? create_cast_from_variant_wrapper(*legacy_variant, json_type) - : create_cast_from_variant_v2_wrapper(json_type); - RETURN_IF_ERROR(to_json(context, intermediate, {0}, 1, rows, null_map)); - RETURN_IF_ERROR(prepare_unpack_dictionaries(context, make_nullable(json_type), - make_nullable(file_type))( - context, intermediate, {1}, 2, rows, nullptr)); - auto output = intermediate.get_by_position(2).column; - const auto& nullable = assert_cast<const ColumnNullable&>(*output); - if (!nullable.has_null()) { - output = nullable.get_nested_column_ptr(); - } - block.get_by_position(result).column = std::move(output); - return Status::OK(); - }; - } - if (to_type->get_primitive_type() == TYPE_VARIANT) { - return [from_type, to_type](FunctionContext* context, Block& block, - const ColumnNumbers& arguments, uint32_t result, size_t rows, - const NullMap::value_type* null_map) { - const auto json_type = std::make_shared<DataTypeJsonb>(); - Block intermediate { - {block.get_by_position(arguments[0]).column->convert_to_full_column_if_const(), - from_type, "file"}, - {nullptr, json_type, "json"}, - {nullptr, block.get_by_position(result).type, "variant"}}; - RETURN_IF_ERROR(create_cast_from_file(from_type, json_type)(context, intermediate, {0}, - 1, rows, null_map)); - const auto* legacy_variant = dynamic_cast<const DataTypeVariant*>(to_type.get()); - auto to_variant = legacy_variant - ? create_cast_to_variant_wrapper(json_type, *legacy_variant) - : create_cast_to_variant_v2_wrapper(json_type); - RETURN_IF_ERROR(to_variant(context, intermediate, {1}, 2, rows, null_map)); - block.get_by_position(result).column = intermediate.get_by_position(2).column; - return Status::OK(); - }; + if (to_type->get_primitive_type() == TYPE_FILE) { + return create_cast_to_file(from_type); } - if (to_type->get_primitive_type() == TYPE_FILE) return create_cast_to_file(from_type); return create_cast_from_file(from_type, to_type); } diff --git a/be/test/exprs/file_expr_test.cpp b/be/test/exprs/file_expr_test.cpp index 6e62a1aa823..b35e4bf1d3e 100644 --- a/be/test/exprs/file_expr_test.cpp +++ b/be/test/exprs/file_expr_test.cpp @@ -41,6 +41,7 @@ #include "exprs/table_function/vexplode_file.h" #include "exprs/vcast_expr.h" #include "exprs/vexpr_context.h" +#include "runtime/runtime_state.h" #include "util/jsonb_document.h" #include "util/jsonb_utils.h" @@ -243,14 +244,9 @@ TEST_F(FileExprTest, StructCastMatchesReorderedPublicNamesAndStringFamily) { TEST_F(FileExprTest, InvalidFileCastReportsErrorsInBothModes) { const auto invalid = public_value("s3://b/bad", integer(-1)); - const auto invalid_json = json( - R"({"uri":"s3://b/bad","offset":null,"size":-1,"content_type":null,"checksum":null,"inline":null})"); for (bool strict : {false, true}) { ColumnPtr output; EXPECT_FALSE(cast(column(public_struct(), {invalid}), file_type, strict, output).ok()); - EXPECT_FALSE(cast(column(std::make_shared<DataTypeJsonb>(), {invalid_json}), - make_nullable(file_type), strict, output) - .ok()); ASSERT_TRUE(try_cast(column(public_struct(), {invalid}), make_nullable(file_type), strict, output) .ok()); @@ -378,129 +374,45 @@ TEST_F(FileExprTest, StructCastAcceptsUntypedNullPublicChildren) { for (size_t i = 1; i < 6; ++i) EXPECT_TRUE(value.get<TYPE_FILE>()[i].is_null()); } -TEST_F(FileExprTest, VariantRoundTripExposesSixFieldsAndPreservesBinaryAndNulls) { - for (const auto& variant_type : - DataTypes {make_nullable(std::make_shared<DataTypeVariant>()), - make_nullable(std::make_shared<DataTypeVariantV2>())}) { - const auto json_type = make_nullable(std::make_shared<DataTypeJsonb>()); - const std::string bytes("a\0\xff", 3); - auto value = file_value("s3://b/variant"); - value.get<TYPE_FILE>()[1] = integer(0); - value.get<TYPE_FILE>()[2] = integer(3); - value.get<TYPE_FILE>()[3] = string("application/octet-stream"); - value.get<TYPE_FILE>()[4] = string("ETAG:opaque"); - value.get<TYPE_FILE>()[5] = Field::create_field<TYPE_VARBINARY>(StringView(bytes)); - const auto input = column(make_nullable(file_type), {value, Field()}); - ColumnPtr variant; - ASSERT_TRUE(cast(input, variant_type, true, variant).ok()); - ASSERT_EQ(variant->size(), 2); - EXPECT_TRUE(variant->is_null_at(1)); - ColumnPtr encoded; - ASSERT_TRUE(cast({variant, variant_type, "variant"}, json_type, true, encoded).ok()); - const auto data = - assert_cast<const ColumnNullable&>(*encoded).get_nested_column().get_data_at(0); - const auto* root = JsonbDocument::createValue(data.data, data.size); - ASSERT_NE(root, nullptr); - ASSERT_TRUE(root->isObject()); - const auto* object = root->unpack<ObjectVal>(); - EXPECT_EQ(object->numElem(), 6); - for (const auto* name : {"uri", "offset", "size", "content_type", "checksum", "inline"}) { - EXPECT_NE(object->find(name), nullptr) << name; - } - ASSERT_TRUE(object->find("inline")->isString()); - EXPECT_EQ(std::string(object->find("inline")->unpack<JsonbStringVal>()->getBlob(), - object->find("inline")->unpack<JsonbStringVal>()->getBlobLen()), - "YQD/"); - EXPECT_TRUE(encoded->is_null_at(1)); - ColumnPtr restored; - ASSERT_TRUE( - cast({variant, variant_type, "variant"}, make_nullable(file_type), true, restored) - .ok()); - const auto roundtrip = (*restored)[0]; - EXPECT_EQ(roundtrip.get<TYPE_FILE>()[0].get<TYPE_STRING>(), "s3://b/variant"); - EXPECT_EQ(roundtrip.get<TYPE_FILE>()[1].get<TYPE_BIGINT>(), 0); - EXPECT_EQ(roundtrip.get<TYPE_FILE>()[2].get<TYPE_BIGINT>(), 3); - EXPECT_EQ(roundtrip.get<TYPE_FILE>()[3].get<TYPE_STRING>(), "application/octet-stream"); - EXPECT_EQ(roundtrip.get<TYPE_FILE>()[4].get<TYPE_STRING>(), "ETAG:opaque"); - EXPECT_EQ(roundtrip.get<TYPE_FILE>()[5].get<TYPE_VARBINARY>().str(), bytes); - EXPECT_TRUE(restored->is_null_at(1)); - EXPECT_EQ((*input.column)[0].get<TYPE_FILE>()[5].get<TYPE_VARBINARY>().str(), bytes); - } -} - -TEST_F(FileExprTest, VariantCastRequiresCompleteObjectAndValidPublicChildren) { - const auto json_type = make_nullable(std::make_shared<DataTypeJsonb>()); +TEST_F(FileExprTest, VariantCastsAreUnsupported) { const auto variant_type = make_nullable(std::make_shared<DataTypeVariantV2>()); + const auto json_type = make_nullable(std::make_shared<DataTypeJsonb>()); const auto input = column( json_type, {json(R"({"checksum":null,"content_type":null,"size":4,"offset":0,"uri":"s3://b/variant","inline":null})"), - json("null"), Field(), json("[]"), json("1"), json(R"("s3://b/variant")"), - json(R"({"offset":null,"size":null,"content_type":null,"checksum":null,"inline":null})"), - json(R"({"uri":"s3://b/variant","size":null,"content_type":null,"checksum":null,"inline":null})"), - json(R"({"uri":"s3://b/variant","offset":null,"content_type":null,"checksum":null,"inline":null})"), - json(R"({"uri":"s3://b/variant","offset":null,"size":null,"checksum":null,"inline":null})"), - json(R"({"uri":"s3://b/variant","offset":null,"size":null,"content_type":null})"), - json(R"({"uri":"s3://b/variant","offset":null,"size":null,"content_type":null,"checksum":null,"inline":"AA"})"), - json(R"({"uri":"s3://b/variant","offset":null,"size":"4","content_type":null,"checksum":null,"inline":null})"), - json(R"({"uri":"s3://b/variant","offset":null,"size":null,"content_type":false,"checksum":null,"inline":null})"), - json(R"({"uri":null,"offset":null,"size":null,"content_type":null,"checksum":null,"inline":null})")}); + json("null"), Field(), json("{}"), json("[]"), json("1"), + json(R"("s3://b/variant")")}); ColumnPtr variant; ASSERT_TRUE(cast(input, variant_type, true, variant).ok()); - ColumnWithTypeAndName source {variant, variant_type, "variant"}; - ColumnPtr output; - EXPECT_FALSE(cast(source, make_nullable(file_type), false, output).ok()); - ASSERT_TRUE(try_cast(source, make_nullable(file_type), false, output).ok()); - ASSERT_EQ(output->size(), input.column->size()); - ASSERT_FALSE(output->is_null_at(0)); - const auto valid = (*output)[0]; - EXPECT_EQ(valid.get<TYPE_FILE>()[2].get<TYPE_BIGINT>(), 4); - for (size_t row = 1; row < output->size(); ++row) EXPECT_TRUE(output->is_null_at(row)) << row; for (bool strict : {false, true}) { - ASSERT_TRUE(try_cast(source, make_nullable(file_type), strict, output).ok()); - EXPECT_FALSE(output->is_null_at(0)); - for (size_t row = 1; row < output->size(); ++row) - EXPECT_TRUE(output->is_null_at(row)) << row; - } - for (size_t row = 0; row < variant->size(); ++row) { - const auto status = cast({variant->cut(row, 1), variant_type, "variant"}, - make_nullable(file_type), true, output); - EXPECT_EQ(status.ok(), row < 3) << row << ": " << status; + ColumnPtr output; + for (size_t row = 0; row < variant->size(); ++row) { + EXPECT_FALSE(cast({variant->cut(row, 1), variant_type, "variant"}, + make_nullable(file_type), strict, output) + .ok()); + } + EXPECT_FALSE(cast(column(file_type, {file_value()}), variant_type, strict, output).ok()); } } -TEST_F(FileExprTest, JsonCastUsesPublicSchemaAndRejectsInvalidValuesAtomically) { +TEST_F(FileExprTest, JsonCastsAreUnsupported) { const auto json_type = std::make_shared<DataTypeJsonb>(); auto input = column( json_type, {json(R"({"uri":"s3://b/key","offset":null,"size":0,"content_type":null,"checksum":null,"inline":null})"), - json("null"), - json(R"({"uri":"s3://b/key","offset":null,"size":null,"content_type":null,"checksum":null,"inline":"AA"})"), - json(R"({"uri":"s3://b/key","offset":1,"size":null,"content_type":null,"checksum":null,"inline":null})"), - json(R"({"uri":"s3://b/key","offset":null,"size":1.5,"content_type":null,"checksum":null,"inline":null})"), - json(R"({"uri":"s3://b/key","offset":null,"size":null,"content_type":null,"checksum":"MD5:ABCDEF0123456789ABCDEF0123456789","inline":null})")}); - ColumnPtr output; - EXPECT_FALSE(cast(input, make_nullable(file_type), false, output).ok()); - ASSERT_TRUE(try_cast(input, make_nullable(file_type), false, output).ok()); - ASSERT_EQ(output->size(), 6); - EXPECT_FALSE(output->is_null_at(0)); - const auto value = (*output)[0]; - EXPECT_EQ(value.get<TYPE_FILE>()[2].get<TYPE_BIGINT>(), 0); - EXPECT_TRUE(value.get<TYPE_FILE>()[5].is_null()); - const auto& nested = assert_cast<const ColumnNullable&>(*output).get_nested_column(); - const auto invalid = nested[2]; - for (const auto& field : invalid.get<TYPE_FILE>()) EXPECT_TRUE(field.is_null()); - for (size_t row = 1; row < 6; ++row) EXPECT_TRUE(output->is_null_at(row)); - EXPECT_FALSE(cast(input, make_nullable(file_type), true, output).ok()); - ASSERT_TRUE( - cast(column(json_type, {json("null")}), make_nullable(file_type), true, output).ok()); - EXPECT_TRUE(output->is_null_at(0)); - ASSERT_TRUE(cast(column(make_nullable(json_type), {Field()}), make_nullable(file_type), true, - output) - .ok()); - EXPECT_TRUE(output->is_null_at(0)); + json("null"), json("{}"), json("[]"), json("1"), json(R"("urn:file")")}); + for (bool strict : {false, true}) { + ColumnPtr output; + for (size_t row = 0; row < input.column->size(); ++row) { + EXPECT_FALSE(cast({input.column->cut(row, 1), json_type, "input"}, + make_nullable(file_type), strict, output) + .ok()); + } + EXPECT_FALSE(cast(column(file_type, {file_value()}), json_type, strict, output).ok()); + } } -TEST_F(FileExprTest, ReverseCastsExposeExactlySixFields) { +TEST_F(FileExprTest, ToJsonAndStructCastExposeExactlySixFields) { const std::string bytes("\0\xff", 2); auto value = file_value("s3://b/key"); value.get<TYPE_FILE>()[5] = Field::create_field<TYPE_VARBINARY>( @@ -516,7 +428,14 @@ TEST_F(FileExprTest, ReverseCastsExposeExactlySixFields) { const auto expected = R"({"uri":"s3://b/key","offset":null,"size":null,"content_type":null,"checksum":null,"inline":"AP8="})"; - ASSERT_TRUE(cast(input, make_nullable(std::make_shared<DataTypeJsonb>()), true, output).ok()); + const auto json_type = make_nullable(std::make_shared<DataTypeJsonb>()); + RuntimeState runtime; + auto context = FunctionContext::create_context(&runtime, json_type, {input.type}); + auto function = SimpleFunctionFactory::instance().get_function("to_json", {input}, json_type); + ASSERT_NE(function, nullptr); + Block block {input, {nullptr, json_type, "json"}}; + ASSERT_TRUE(function->execute(context.get(), block, {0}, 1, input.column->size()).ok()); + output = block.get_by_position(1).column; const auto& json_column = assert_cast<const ColumnNullable&>(*output).get_nested_column(); const auto encoded = json_column.get_data_at(0); EXPECT_EQ(JsonbToJson::jsonb_to_json_string(encoded.data, encoded.size), expected); @@ -525,6 +444,35 @@ TEST_F(FileExprTest, ReverseCastsExposeExactlySixFields) { EXPECT_EQ((*input.column)[0].get<TYPE_FILE>()[5].get<TYPE_VARBINARY>().size(), bytes.size()); } +TEST_F(FileExprTest, ToJsonPreservesConstantFileAndNull) { + const auto type = make_nullable(file_type); + const auto json_type = make_nullable(std::make_shared<DataTypeJsonb>()); + auto value = file_value("urn:constant"); + value.get<TYPE_FILE>()[5] = Field::create_field<TYPE_VARBINARY>(StringView()); + for (bool is_null : {false, true}) { + const ColumnWithTypeAndName input {type->create_column_const(3, is_null ? Field() : value), + type, "input"}; + RuntimeState runtime; + auto context = FunctionContext::create_context(&runtime, json_type, {type}); + auto function = + SimpleFunctionFactory::instance().get_function("to_json", {input}, json_type); + ASSERT_NE(function, nullptr); + Block block {input, {nullptr, json_type, "json"}}; + ASSERT_TRUE(function->execute(context.get(), block, {0}, 1, 3).ok()); + const auto& output = block.get_by_position(1).column; + ASSERT_EQ(output->size(), 3); + for (size_t row = 0; row < 3; ++row) { + EXPECT_EQ(output->is_null_at(row), is_null); + if (!is_null) { + const auto encoded = output->get_data_at(row); + EXPECT_EQ( + JsonbToJson::jsonb_to_json_string(encoded.data, encoded.size), + R"({"uri":"urn:constant","offset":null,"size":null,"content_type":null,"checksum":null,"inline":""})"); + } + } + } +} + TEST_F(FileExprTest, ReverseStructMatchesPublicNamesAndRequiresNullableChildren) { const auto& file = assert_cast<const DataTypeFile&>(*file_type); ColumnPtr output; @@ -619,35 +567,6 @@ TEST_F(FileExprTest, GetterRejectsDynamicNumericNullAndUnknownSelectors) { } } -TEST_F(FileExprTest, JsonCastRejectsMissingDuplicateUnknownWrongTypeAndOutOfRangeFields) { - const auto type = std::make_shared<DataTypeJsonb>(); - const auto input = column( - type, - {json(R"({"uri":"s3://b/a"})"), - json(R"({"uri":"s3://b/a","offset":null,"size":null,"content_type":null})"), - json(R"({"uri":"s3://b/a","uri":"s3://b/b","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null})"), - json(R"({"URI":"s3://b/a","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null})"), - json(R"({"uri":"s3://b/a","offset":null,"size":9223372036854775808,"content_type":null,"checksum":null,"inline":null})"), - json(R"({"uri":"s3://b/a","offset":9223372036854775807,"size":1,"content_type":null,"checksum":null,"inline":null})"), - json(R"({"uri":"s3://b/a","offset":null,"size":null,"content_type":null,"checksum":"md5:0123456789abcdef0123456789abcdef","inline":null})"), - json(R"({"uri":"s3://b/a","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null,"other":null})"), - json(R"({"uri":1,"offset":null,"size":null,"content_type":null,"checksum":null,"inline":null})"), - json(R"({"uri":"s3://b/a","offset":null,"size":"12","content_type":null,"checksum":null,"inline":null})"), - json(R"({"uri":"s3://b/a","offset":null,"size":null,"content_type":false,"checksum":null,"inline":null})"), - json("[]"), json("1")}); - ColumnPtr output; - EXPECT_FALSE(cast(input, make_nullable(file_type), false, output).ok()); - ASSERT_TRUE(try_cast(input, make_nullable(file_type), false, output).ok()); - for (size_t row = 0; row < input.column->size(); ++row) - EXPECT_TRUE(output->is_null_at(row)) << row; - for (size_t row = 0; row < input.column->size(); ++row) { - EXPECT_FALSE(cast({input.column->cut(row, 1), type, "input"}, make_nullable(file_type), - true, output) - .ok()) - << row; - } -} - TEST_F(FileExprTest, GetterSupportsConstantFileAndConstantNullableSelector) { auto input = file_type->create_column_const(4, file_value("s3://b/key")); auto name = make_nullable(text)->create_column_const(4, string("URI")); @@ -936,19 +855,12 @@ TEST_F(FileExprTest, TryCastMakesInvalidFileNullInBothSessionModes) { } TEST_F(FileExprTest, NestedTryCastUsesGenericFailureHandling) { - for (bool from_json : {false, true}) { - DataTypePtr leaf = - from_json ? DataTypePtr(std::make_shared<DataTypeJsonb>()) : public_struct(); + { + DataTypePtr leaf = public_struct(); leaf = make_nullable(leaf); const auto target_leaf = make_nullable(file_type); - const auto good = - from_json - ? json(R"({"uri":"s3://b/good","offset":null,"size":4,"content_type":null,"checksum":null,"inline":null})") - : public_value("s3://b/good", integer(4)); - const auto bad = - from_json - ? json(R"({"uri":"s3://b/bad","offset":null,"size":-1,"content_type":null,"checksum":null,"inline":null})") - : public_value("s3://b/bad", integer(-1)); + const auto good = public_value("s3://b/good", integer(4)); + const auto bad = public_value("s3://b/bad", integer(-1)); const auto array_source = make_nullable(std::make_shared<DataTypeArray>(leaf)); const auto array_target = make_nullable(std::make_shared<DataTypeArray>(target_leaf)); const auto mixed_array = Field::create_field<TYPE_ARRAY>(Array {good, bad, Field()}); @@ -1007,65 +919,44 @@ TEST_F(FileExprTest, NestedTryCastUsesGenericFailureHandling) { } } -TEST_F(FileExprTest, JsonTryCastUsesGenericFailureHandling) { +TEST_F(FileExprTest, JsonCastsRejectNestedFileTypes) { const auto source = std::make_shared<DataTypeJsonb>(); const auto arrays = column( source, - {json(R"([{"uri":"s3://b/good","offset":null,"size":4,"content_type":null,"checksum":null,"inline":null}, - {"uri":"s3://b/bad","offset":null,"size":-1,"content_type":null,"checksum":null,"inline":null},null])"), + {json(R"([{"uri":"s3://b/good","offset":null,"size":4,"content_type":null,"checksum":null,"inline":null},null])"), json("[]"), json("null")}); const auto array_target = make_nullable(std::make_shared<DataTypeArray>(make_nullable(file_type))); const auto structs = column( source, - {json(R"({"file":{"uri":"s3://b/bad","offset":null,"size":-1,"content_type":null,"checksum":null,"inline":null},"label":"kept"})"), + {json(R"({"file":{"uri":"s3://b/first","offset":null,"size":1,"content_type":null,"checksum":null,"inline":null},"label":"kept"})"), json(R"({"file":{"uri":"s3://b/good","offset":null,"size":4,"content_type":null,"checksum":null,"inline":null},"label":"also"})")}); const auto struct_target = make_nullable(std::make_shared<DataTypeStruct>( DataTypes {make_nullable(file_type), make_nullable(text)}, Strings {"file", "label"})); for (bool strict : {false, true}) { SCOPED_TRACE(strict ? "strict" : "non-strict"); ColumnPtr output; - ASSERT_TRUE(try_cast(arrays, array_target, strict, output).ok()); - if (strict) { - EXPECT_TRUE(output->is_null_at(0)); - } else { - ASSERT_FALSE(output->is_null_at(0)); - EXPECT_EQ((*output)[0].get<TYPE_ARRAY>()[0].get<TYPE_FILE>()[2].get<TYPE_BIGINT>(), 4); - EXPECT_TRUE((*output)[0].get<TYPE_ARRAY>()[1].is_null()); - EXPECT_TRUE((*output)[0].get<TYPE_ARRAY>()[2].is_null()); - } - EXPECT_TRUE((*output)[1].get<TYPE_ARRAY>().empty()); - EXPECT_TRUE(output->is_null_at(2)); - ASSERT_TRUE(try_cast(structs, struct_target, strict, output).ok()); - if (strict) { - EXPECT_TRUE(output->is_null_at(0)); - } else { - ASSERT_FALSE(output->is_null_at(0)); - EXPECT_TRUE((*output)[0].get<TYPE_STRUCT>()[0].is_null()); - EXPECT_EQ((*output)[0].get<TYPE_STRUCT>()[1].get<TYPE_STRING>(), "kept"); - } - EXPECT_EQ((*output)[1].get<TYPE_STRUCT>()[0].get<TYPE_FILE>()[2].get<TYPE_BIGINT>(), 4); + EXPECT_FALSE(cast(arrays, array_target, strict, output).ok()); + EXPECT_FALSE(cast(structs, struct_target, strict, output).ok()); + const auto files = + column(array_target, {Field::create_field<TYPE_ARRAY>(Array {file_value()})}); + EXPECT_FALSE(cast(files, make_nullable(source), strict, output).ok()); } } -TEST_F(FileExprTest, JsonTryCastKeepsExistingNonFileFailureBehavior) { +TEST_F(FileExprTest, JsonCastsKeepExistingNonFileBehavior) { const auto source = std::make_shared<DataTypeJsonb>(); const auto target = make_nullable(std::make_shared<DataTypeStruct>( - DataTypes {make_nullable(file_type), make_nullable(bigint)}, - Strings {"file", "number"})); - const auto input = column( - source, - {json(R"({"file":{"uri":"s3://b/bad","offset":null,"size":-1,"content_type":null,"checksum":null,"inline":null},"number":7})"), - json(R"({"file":{"uri":"s3://b/good","offset":null,"size":4,"content_type":null,"checksum":null,"inline":null},"number":{}})"), - json(R"({"file":{"uri":"s3://b/good","offset":null,"size":4,"content_type":null,"checksum":null,"inline":null},"number":8})")}); + DataTypes {make_nullable(text), make_nullable(bigint)}, Strings {"label", "number"})); + const auto input = column(source, {json(R"({"label":"first","number":7})"), + json(R"({"label":"second","number":8})")}); ColumnPtr output; - ASSERT_TRUE(try_cast(input, target, true, output).ok()); - ASSERT_EQ(output->size(), 3); - EXPECT_TRUE(output->is_null_at(0)); - EXPECT_TRUE(output->is_null_at(1)); - ASSERT_FALSE(output->is_null_at(2)); - EXPECT_EQ((*output)[2].get<TYPE_STRUCT>()[0].get<TYPE_FILE>()[2].get<TYPE_BIGINT>(), 4); - EXPECT_EQ((*output)[2].get<TYPE_STRUCT>()[1].get<TYPE_BIGINT>(), 8); + ASSERT_TRUE(cast(input, target, true, output).ok()); + ASSERT_EQ(output->size(), 2); + EXPECT_EQ((*output)[0].get<TYPE_STRUCT>()[0].get<TYPE_STRING>(), "first"); + EXPECT_EQ((*output)[0].get<TYPE_STRUCT>()[1].get<TYPE_BIGINT>(), 7); + EXPECT_EQ((*output)[1].get<TYPE_STRUCT>()[0].get<TYPE_STRING>(), "second"); + EXPECT_EQ((*output)[1].get<TYPE_STRUCT>()[1].get<TYPE_BIGINT>(), 8); } TEST_F(FileExprTest, NestedTryCastKeepsExistingNonFileFailureBehavior) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java index b47acbdb001..cd878dd5fb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/check/CheckCast.java @@ -362,7 +362,8 @@ public class CheckCast implements ExpressionPatternRuleFactory { if (originalType.isFileType() || targetType.isFileType()) { return checkFileCast(originalType, targetType); } - if ((originalType.isStringLikeType() || originalType.isVariantType() || targetType.isVariantType()) + if ((originalType.isJsonType() || targetType.isJsonType() || originalType.isStringLikeType() + || originalType.isVariantType() || targetType.isVariantType()) && (originalType.typeContainsFile() || targetType.typeContainsFile())) { return false; } @@ -455,9 +456,6 @@ public class CheckCast implements ExpressionPatternRuleFactory { return true; } DataType publicType = source.isFileType() ? target : source; - if (publicType.isJsonType() || publicType.isVariantType()) { - return true; - } if (!(publicType instanceof StructType)) { return false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java index a6e571c2323..58623872ac6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ToJson.java @@ -96,7 +96,7 @@ public class ToJson extends ScalarFunction @Override public List<FunctionSignature> getSignatures() { DataType firstChildType = child(0).getDataType(); - if (firstChildType.isStructType() || firstChildType.isArrayType()) { + if (firstChildType.isStructType() || firstChildType.isArrayType() || firstChildType.isFileType()) { return ImmutableList.of(FunctionSignature.ret(JsonType.INSTANCE).args(firstChildType)); } if (firstChildType.isMapType()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java index e6b6a958a95..89d8ff388f6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/types/FileTypeExpressionTest.java @@ -122,7 +122,7 @@ public class FileTypeExpressionTest { } for (String sql : Arrays.asList("select 1", "select named_struct('s', 'text')", "select element_at(" + file + ", 'uri')", - "select cast(" + file + " as json)")) { + "select to_json(" + file + ")")) { Assertions.assertDoesNotThrow(() -> PlanChecker.from(flight).analyze(sql), sql); } } finally { @@ -260,13 +260,37 @@ public class FileTypeExpressionTest { } } + @Test + public void testFileJsonCastsAreUnsupported() { + for (boolean strict : Arrays.asList(true, false)) { + for (String wrapper : Arrays.asList("%s", "ARRAY<%s>", "STRUCT<f:%s>", "MAP<STRING,%s>")) { + DataType file = type(String.format(wrapper, "FILE")); + DataType json = type(String.format(wrapper, "JSON")); + Assertions.assertFalse(CheckCast.check(file, json, strict)); + Assertions.assertFalse(CheckCast.check(json, file, strict)); + Assertions.assertFalse(CheckCast.check(file, JsonType.INSTANCE, strict)); + Assertions.assertFalse(CheckCast.check(JsonType.INSTANCE, file, strict)); + } + } + } + + @Test + public void testToJsonAcceptsFile() { + for (String value : Arrays.asList("cast(null as file)", + "cast(named_struct('uri', 'urn:x', 'offset', null, 'size', null, " + + "'content_type', null, 'checksum', null, 'inline', null) as file)")) { + Plan plan = PlanChecker.from(MemoTestUtils.createConnectContext()) + .analyze("select to_json(" + value + ")").getPlan(); + Assertions.assertEquals(JsonType.INSTANCE, plan.getOutput().get(0).getDataType()); + } + } + @Test public void testFileCastUsesOrdinaryTypeChecksAndNullLiteralCoercion() { Assertions.assertFalse(Cast.castNullable(false, FileType.INSTANCE.publicStructType(), FileType.INSTANCE)); Assertions.assertTrue(Cast.castNullable(true, FileType.INSTANCE.publicStructType(), FileType.INSTANCE)); Assertions.assertTrue(Cast.castNullable(false, JsonType.INSTANCE, FileType.INSTANCE)); - Assertions.assertDoesNotThrow(() -> new Cast(new SlotReference("j", JsonType.INSTANCE), - FileType.INSTANCE).checkLegalityBeforeTypeCoercion()); + Assertions.assertFalse(CheckCast.check(JsonType.INSTANCE, FileType.INSTANCE, true)); Expression converted = TypeCoercionUtils.castIfNotSameType(new NullLiteral(FileType.INSTANCE), JsonType.INSTANCE); Assertions.assertTrue(converted instanceof NullLiteral); @@ -281,7 +305,7 @@ public class FileTypeExpressionTest { String metadata = "uri:STRING,offset:BIGINT,size:BIGINT,content_type:STRING,checksum:STRING"; String fields = metadata + ",inline:VARBINARY"; for (boolean strict : Arrays.asList(true, false)) { - for (String source : Arrays.asList("STRUCT<" + fields + ">", "JSON", "VARIANT", + for (String source : Arrays.asList("STRUCT<" + fields + ">", "STRUCT<inline:VARBINARY,checksum:VARCHAR(10),size:BIGINT,uri:CHAR(10)," + "content_type:STRING,offset:BIGINT>")) { DataType value = type(source); @@ -290,7 +314,7 @@ public class FileTypeExpressionTest { Assertions.assertTrue(TypeCoercionUtils.implicitCast(value, FileType.INSTANCE).isPresent()); Assertions.assertTrue(TypeCoercionUtils.implicitCast(FileType.INSTANCE, value).isPresent()); } - for (String source : Arrays.asList("STRING", "INT", "STRUCT<uri:STRING>", + for (String source : Arrays.asList("STRING", "JSON", "VARIANT", "INT", "STRUCT<uri:STRING>", "STRUCT<" + metadata + ">", "STRUCT<" + metadata + ",inline:STRING>", "STRUCT<" + fields + ",extra:STRING>", @@ -344,9 +368,9 @@ public class FileTypeExpressionTest { Assertions.assertFalse(CheckCast.check(type(String.format(wrapper, leaf)), type(String.format(wrapper, "ARRAY<FILE>")), strict), wrapper + leaf); } - Assertions.assertTrue(CheckCast.check(type(String.format(wrapper, "VARIANT")), + Assertions.assertFalse(CheckCast.check(type(String.format(wrapper, "VARIANT")), type(String.format(wrapper, "FILE")), strict), wrapper); - Assertions.assertTrue(CheckCast.check(type(String.format(wrapper, "FILE")), + Assertions.assertFalse(CheckCast.check(type(String.format(wrapper, "FILE")), type(String.format(wrapper, "VARIANT")), strict), wrapper); } Assertions.assertTrue(CheckCast.check(VariantType.INSTANCE, type("ARRAY<INT>"), strict)); @@ -634,7 +658,8 @@ public class FileTypeExpressionTest { "select cast(named_struct('uri', 'urn:x', 'offset', cast(0 as bigint), 'size', cast(0 as " + "bigint), 'content_type', '', 'checksum', '', " + "'inline', null) as file) union all " - + "select cast(cast('{\"uri\":\"urn:y\"}' as json) as file) " + + "select cast(named_struct('uri', 'urn:y', 'offset', null, 'size', null, " + + "'content_type', null, 'checksum', null, 'inline', null) as file) " + "union all select cast(null as file)", "select cast(null as array<file>) union all " + "select cast(array(named_struct('uri', 'urn:x', 'offset', cast(0 as bigint), 'size', " @@ -679,11 +704,10 @@ public class FileTypeExpressionTest { "select cast(named_struct('uri', 'urn:x', 'offset', cast(0 as bigint), 'size', cast(0 as " + "bigint), 'content_type', '', 'checksum', '', " + "'inline', null) as file) " - + "union all select cast(cast('{}' as json) as file) union all select cast(null as file)", + + "union all select cast(null as file)", "select cast(null as file) union all select cast(named_struct('uri', 'urn:x', 'offset', cast(0 " + "as bigint), 'size', cast(0 as bigint), 'content_type', '', 'checksum', '', " + "'inline', null) as file) " - + "union all select cast(cast('{}' as json) as file) " + "union all select cast(named_struct('uri', 'urn:z', 'offset', cast(0 as bigint), " + "'size', cast(0 as bigint), 'content_type', '', 'checksum', '', " + "'inline', null) as file)")) { diff --git a/regression-test/data/datatype_p0/complex_types/test_file_type.out b/regression-test/data/datatype_p0/complex_types/test_file_type.out index 72a8ecad686..5ab79481012 100644 --- a/regression-test/data/datatype_p0/complex_types/test_file_type.out +++ b/regression-test/data/datatype_p0/complex_types/test_file_type.out @@ -7,6 +7,17 @@ 2 \N \N \N 3 {"uri":"urn:example:three","offset":2,"size":8,"content_type":null,"checksum":null,"inline":""} {"uri":"urn:example:three","offset":2,"size":8,"content_type":null,"checksum":null,"inline":""} {"uri":"urn:example:three","offset":2,"size":8,"content_type":null,"checksum":null,"inline":""} +-- !to_json_nested -- +1 [{"uri":"S3://Bucket/a%2fb?versionId=v1","offset":null,"size":0,"content_type":"Image/PNG","checksum":"CRC32:00000000","inline":"AP+A"},null] {"f":{"uri":"S3://Bucket/a%2fb?versionId=v1","offset":null,"size":0,"content_type":"Image/PNG","checksum":"CRC32:00000000","inline":"AP+A"}} {"source":{"uri":"S3://Bucket/a%2fb?versionId=v1","offset":null,"size":0,"content_type":"Image/PNG","checksum":"CRC32:00000000","inline":"AP+A"}} +2 [null,null] {"f":null} {"source":null} +3 [{"uri":"urn:example:three","offset":2,"size":8,"content_type":null,"checksum":null,"inline":""},null] {"f":{"uri":"urn:example:three","offset":2,"size":8,"content_type":null,"checksum":null,"inline":""}} {"source":{"uri":"urn:example:three","offset":2,"size":8,"content_type":null,"checksum":null,"inline":""}} + +-- !to_json_inline -- +1 {"uri":null,"offset":null,"size":null,"content_type":null,"checksum":null,"inline":"AP8="} +2 {"uri":null,"offset":null,"size":null,"content_type":null,"checksum":null,"inline":""} +3 \N +4 {"uri":"urn:reference","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null} + -- !fields -- 1 S3://Bucket/a%2fb?versionId=v1 \N 0 Image/PNG CRC32:00000000 S3://Bucket/a%2fb?versionId=v1 00FF80 00FF80 false false true 2 \N \N \N \N \N \N \N \N true true false @@ -129,7 +140,7 @@ relative/path relative/path s3://bucket/a b+中文%ZZ#fragment s3://bucket/a b+中文%ZZ#fragment -- !try_cast -- -true true true +true true -- !nested_try_cast -- \N \N \N @@ -137,11 +148,6 @@ true true true -- !explicit_reverse -- {"uri":"urn:cast","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null} {"uri":"urn:cast","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null} urn:cast {"uri":"urn:cast","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null} urn:cast --- !variant_casts -- -1 S3://Bucket/a%2fb?versionId=v1 0 00FF80 00FF80 -2 \N \N \N \N -3 urn:example:three 8 - -- !inline_struct_roundtrip -- 1 00FF80 2 \N @@ -149,7 +155,7 @@ true true true -- !explicit_forward -- 2 urn:struct urn:struct -3 urn:json urn:json +3 urn:reference urn:reference -- !inline_values -- 1 {"uri":null,"offset":null,"size":null,"content_type":null,"checksum":null,"inline":"AP8="} \N \N 00FF false diff --git a/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy b/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy index 3e33e3e199d..beb91d4f868 100644 --- a/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy +++ b/regression-test/suites/datatype_p0/complex_types/test_file_type.groovy @@ -18,15 +18,14 @@ suite("test_file_type") { def publicStruct = "STRUCT<uri:VARCHAR(65533),offset:BIGINT,size:BIGINT," + "content_type:VARCHAR(1024),checksum:VARCHAR(1024),inline:VARBINARY>" - // Only scalar JSON rows are combined here; FILE is constructed after UNION ALL. def fileCtes = """ WITH test_file_type_values AS ( SELECT id, CAST(j AS FILE) AS f FROM ( - SELECT 1 AS id, JSON_PARSE('{"uri":"S3://Bucket/a%2fb?versionId=v1","size":0, - "content_type":"Image/PNG","checksum":"CRC32:00000000","offset":null,"inline":"AP+A"}') AS j + SELECT 1 AS id, NAMED_STRUCT('uri', 'S3://Bucket/a%2fb?versionId=v1', 'size', CAST(0 AS BIGINT), + 'content_type', 'Image/PNG', 'checksum', 'CRC32:00000000', 'offset', CAST(NULL AS BIGINT), 'inline', X'00FF80') AS j UNION ALL SELECT 2, NULL - UNION ALL SELECT 3, JSON_PARSE('{"uri":"urn:example:three","offset":2,"size":8, - "content_type":null,"checksum":null,"inline":""}') + UNION ALL SELECT 3, NAMED_STRUCT('uri', 'urn:example:three', 'size', CAST(8 AS BIGINT), + 'content_type', NULL, 'checksum', NULL, 'offset', CAST(2 AS BIGINT), 'inline', X'') ) scalar_rows ), test_file_type_nested AS ( SELECT id, ARRAY(f, NULL) AS files, @@ -34,13 +33,13 @@ suite("test_file_type") { FROM test_file_type_values ), test_file_type_inline_values AS ( SELECT id, CAST(j AS FILE) AS f FROM ( - SELECT 1 AS id, JSON_PARSE('{"uri":null,"offset":null,"size":null, - "content_type":null,"checksum":null,"inline":"AP8="}') AS j - UNION ALL SELECT 2, JSON_PARSE('{"uri":null,"offset":null,"size":null, - "content_type":null,"checksum":null,"inline":""}') + SELECT 1 AS id, NAMED_STRUCT('uri', CAST(NULL AS STRING), 'offset', NULL, 'size', NULL, + 'content_type', NULL, 'checksum', NULL, 'inline', X'00FF') AS j + UNION ALL SELECT 2, NAMED_STRUCT('uri', NULL, 'offset', NULL, 'size', NULL, + 'content_type', NULL, 'checksum', NULL, 'inline', X'') UNION ALL SELECT 3, NULL - UNION ALL SELECT 4, JSON_PARSE('{"uri":"urn:reference","offset":null,"size":null, - "content_type":null,"checksum":null,"inline":null}') + UNION ALL SELECT 4, NAMED_STRUCT('uri', 'urn:reference', 'offset', NULL, 'size', NULL, + 'content_type', NULL, 'checksum', NULL, 'inline', NULL) ) scalar_inline_rows ) """ @@ -50,7 +49,12 @@ suite("test_file_type") { 'URI', 'urn:example:three', 'OFFSET', CAST(CAST(2 AS INT) AS BIGINT), 'content_type', NULL, 'checksum', NULL, 'inline', X'') AS FILE) """) - qt_values fileQuery("SELECT id, f, CAST(CAST(f AS JSON) AS STRING), CAST(f AS JSON) FROM test_file_type_values ORDER BY id") + qt_values fileQuery("SELECT id, f, CAST(TO_JSON(f) AS STRING), TO_JSON(f) FROM test_file_type_values ORDER BY id") + qt_to_json_nested fileQuery(""" + SELECT id, TO_JSON(files), TO_JSON(holder), TO_JSON(lookup) + FROM test_file_type_nested ORDER BY id + """) + qt_to_json_inline fileQuery("SELECT id, TO_JSON(f) FROM test_file_type_inline_values ORDER BY id") qt_fields fileQuery(""" SELECT id, ELEMENT_AT(f, 'uri'), ELEMENT_AT(f, 'offset'), ELEMENT_AT(f, 'size'), ELEMENT_AT(f, 'content_type'), ELEMENT_AT(f, 'checksum'), ELEMENT_AT(f, 'UrI'), @@ -140,8 +144,8 @@ suite("test_file_type") { } } } - def inlineFallback = """CAST(JSON_PARSE('{"uri":null,"offset":null,"size":null, - "content_type":null,"checksum":null,"inline":"AP+A"}') AS FILE)""" + def inlineFallback = """CAST(NAMED_STRUCT('uri', NULL, 'offset', NULL, 'size', NULL, + 'content_type', NULL, 'checksum', NULL, 'inline', X'00FF80') AS FILE)""" qt_conditional_values fileQuery(""" SELECT id, COALESCE(f, ${inlineFallback}), IF(id = 2, ${inlineFallback}, f), IFNULL(f, ${inlineFallback}), CASE WHEN id = 2 THEN ${inlineFallback} ELSE f END, @@ -317,27 +321,41 @@ suite("test_file_type") { UNION ALL SELECT 's3://bucket/a b+中文%ZZ#fragment') t ORDER BY uri """ sql "SET enable_strict_cast = true" - for (def invalid in [ - '{"uri":"urn:missing-inline","offset":null,"size":null,"content_type":null,"checksum":null}', - '{"uri":"urn:bad-inline","offset":null,"size":null,"content_type":null,"checksum":null,"inline":"AB=="}', - '{"uri":"urn:bad-inline","offset":null,"size":null,"content_type":null,"checksum":null,"inline":"AA"}', - '{"uri":"urn:x","offset":1,"size":null,"content_type":null,"checksum":null,"inline":null}', - '{"uri":"urn:x","size":-1,"offset":null,"content_type":null,"checksum":null,"inline":null}', - '{"uri":"urn:x","offset":9223372036854775807,"size":1,"content_type":null,"checksum":null,"inline":null}', - '{"uri":"urn:x","size":1.5,"offset":null,"content_type":null,"checksum":null,"inline":null}', '{"uri":"urn:x","content_type":"invalid","offset":null,"size":null,"checksum":null,"inline":null}', - '{"uri":"urn:x","checksum":"md5:00000000000000000000000000000000","offset":null,"size":null,"content_type":null,"inline":null}', - '{"uri":"urn:x","checksum":"MD5:ABCDEF00000000000000000000000000","offset":null,"size":null,"content_type":null,"inline":null}', - '{"uri":"urn:x","inline":7,"offset":null,"size":null,"content_type":null,"checksum":null}', '{"URI":"urn:x","uri":null,"offset":null,"size":null,"content_type":null,"checksum":null,"inline":null}', - '{"uri":"urn:x","uri":"urn:y","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null}', '{"uri":"urn:x","extra":1,"offset":null,"size":null,"content_type":null,"checksum":null,"inline":null}' - ]) { - test { - sql fileQuery("SELECT CAST(JSON_PARSE('${invalid}') AS FILE)") - exception "FILE" + for (def strict in [true, false]) { + sql "SET enable_strict_cast = ${strict}" + for (def castFunction in ["CAST", "TRY_CAST"]) { + for (def externalType in ["JSON", "VARIANT"]) { + for (def expression in [ + "${castFunction}(f AS ${externalType})", + "${castFunction}(CAST(TO_JSON(f) AS ${externalType}) AS FILE)", + "${castFunction}(CAST(NULL AS FILE) AS ${externalType})", + "${castFunction}(CAST(NULL AS ${externalType}) AS FILE)", + "${castFunction}(ARRAY(f) AS ARRAY<${externalType}>)", + "${castFunction}(CAST(TO_JSON(ARRAY(f)) AS ARRAY<${externalType}>) AS ARRAY<FILE>)", + "${castFunction}(CAST(TO_JSON(ARRAY(f)) AS ${externalType}) AS ARRAY<FILE>)", + "${castFunction}(ARRAY(f) AS ${externalType})", + "${castFunction}(NAMED_STRUCT('f', f) AS ${externalType})", + "${castFunction}(CAST(TO_JSON(NAMED_STRUCT('f', f)) AS ${externalType}) AS STRUCT<f:FILE>)", + "${castFunction}(MAP('f', f) AS ${externalType})", + "${castFunction}(CAST(TO_JSON(MAP('f', f)) AS ${externalType}) AS MAP<STRING,FILE>)", + "${castFunction}(NAMED_STRUCT('f', f) AS STRUCT<f:${externalType}>)", + "${castFunction}(CAST(TO_JSON(NAMED_STRUCT('f', f)) AS STRUCT<f:${externalType}>) AS STRUCT<f:FILE>)", + "${castFunction}(MAP('f', f) AS MAP<STRING,${externalType}>)", + "${castFunction}(CAST(NULL AS MAP<STRING,${externalType}>) AS MAP<STRING,FILE>)" + ]) { + test { + sql fileQuery("SELECT ${expression} FROM test_file_type_values") + exception "FILE" + } + } + } } } + sql "SET enable_strict_cast = true" qt_try_cast fileQuery(""" - SELECT TRY_CAST(JSON_PARSE('{"uri":"relative/path","offset":null,"size":-1,"content_type":null,"checksum":null,"inline":null}') AS FILE) IS NULL, - CAST(JSON_PARSE('null') AS FILE) IS NULL, CAST(NULL AS FILE) IS NULL + SELECT TRY_CAST(NAMED_STRUCT('uri', 'relative/path', 'offset', NULL, 'size', CAST(-1 AS BIGINT), + 'content_type', NULL, 'checksum', NULL, 'inline', NULL) AS FILE) IS NULL, + CAST(NULL AS FILE) IS NULL """) qt_nested_try_cast fileQuery(""" SELECT TRY_CAST(ARRAY(NAMED_STRUCT('uri', 'urn:valid', 'offset', NULL, 'size', NULL, 'content_type', NULL, 'checksum', NULL, 'inline', NULL), @@ -350,7 +368,8 @@ suite("test_file_type") { """) sql "SET enable_strict_cast = false" test { - sql fileQuery("""SELECT CAST(JSON_PARSE('{"uri":"urn:x","size":-1,"offset":null,"content_type":null,"checksum":null,"inline":null}') AS FILE)""") + sql fileQuery("""SELECT CAST(NAMED_STRUCT('uri', 'urn:x', 'size', CAST(-1 AS BIGINT), 'offset', NULL, + 'content_type', NULL, 'checksum', NULL, 'inline', NULL) AS FILE)""") exception "FILE size must be nonnegative" } sql "SET enable_strict_cast = true" @@ -415,9 +434,9 @@ suite("test_file_type") { def fileValue = "CAST(NAMED_STRUCT('uri', 'urn:cast', 'offset', NULL, 'size', NULL, 'content_type', NULL, 'checksum', NULL, 'inline', NULL) AS FILE)" def fileArrayValue = "CAST(ARRAY(CAST(${fileValue} AS ${publicStruct})) AS ARRAY<FILE>)" qt_explicit_reverse fileQuery(""" - SELECT CAST(CAST(${fileValue} AS JSON) AS STRING), CAST(${fileValue} AS JSON), + SELECT CAST(TO_JSON(${fileValue}) AS STRING), TO_JSON(${fileValue}), ELEMENT_AT(CAST(${fileValue} AS ${publicStruct}), 'uri'), - ELEMENT_AT(CAST(CAST(${fileArrayValue} AS ARRAY<JSON>) AS ARRAY<STRING>), 1), + CAST(TO_JSON(ELEMENT_AT(${fileArrayValue}, 1)) AS STRING), ELEMENT_AT(ELEMENT_AT(CAST(${fileArrayValue} AS ARRAY<${publicStruct}>), 1), 'uri') """) for (def query in [ @@ -431,13 +450,6 @@ suite("test_file_type") { exception "FILE" } } - qt_variant_casts fileQuery(""" - SELECT id, ELEMENT_AT(CAST(CAST(f AS VARIANT) AS FILE), 'uri'), - ELEMENT_AT(CAST(CAST(f AS VARIANT) AS FILE), 'size'), - FROM_BINARY(ELEMENT_AT(CAST(CAST(f AS VARIANT) AS FILE), 'inline')), - FROM_BINARY(ELEMENT_AT(CAST(CAST(f AS JSON) AS FILE), 'inline')) - FROM test_file_type_values ORDER BY id - """) qt_inline_struct_roundtrip fileQuery(""" SELECT id, FROM_BINARY(ELEMENT_AT(CAST(CAST(f AS STRUCT<inline:VARBINARY,checksum:STRING,content_type:STRING,size:BIGINT,offset:BIGINT,uri:STRING>) @@ -448,10 +460,10 @@ suite("test_file_type") { SELECT id, ELEMENT_AT(CAST(j AS FILE), 'uri'), ELEMENT_AT(ELEMENT_AT(CAST(ARRAY(CAST(CAST(j AS FILE) AS ${publicStruct})) AS ARRAY<FILE>), 1), 'uri') FROM ( - SELECT 2 AS id, JSON_PARSE('{"uri":"urn:struct","offset":null,"size":null, - "content_type":null,"checksum":null,"inline":null}') AS j - UNION ALL SELECT 3, JSON_PARSE('{"uri":"urn:json","offset":null,"size":null, - "content_type":null,"checksum":null,"inline":null}') + SELECT 2 AS id, NAMED_STRUCT('uri', 'urn:struct', 'offset', NULL, 'size', NULL, + 'content_type', NULL, 'checksum', NULL, 'inline', NULL) AS j + UNION ALL SELECT 3, NAMED_STRUCT('uri', 'urn:reference', 'offset', NULL, 'size', NULL, + 'content_type', NULL, 'checksum', NULL, 'inline', NULL) ) scalar_cast_rows ORDER BY id """) diff --git a/regression-test/suites/datatype_p0/complex_types/test_file_type_optimizer.groovy b/regression-test/suites/datatype_p0/complex_types/test_file_type_optimizer.groovy index 5a21b195e1c..17a8c0b3375 100644 --- a/regression-test/suites/datatype_p0/complex_types/test_file_type_optimizer.groovy +++ b/regression-test/suites/datatype_p0/complex_types/test_file_type_optimizer.groovy @@ -20,8 +20,8 @@ suite("test_file_type_optimizer") { "content_type:VARCHAR(1024),checksum:VARCHAR(1024),inline:VARBINARY>" def fileCtes = """ WITH file_values AS ( - SELECT number AS id, CAST(JSON_PARSE(CONCAT('{"uri":"urn:optimizer:', number, - '","offset":null,"size":null,"content_type":null,"checksum":null,"inline":null}')) AS FILE) AS f + SELECT number AS id, CAST(NAMED_STRUCT('uri', CONCAT('urn:optimizer:', number), + 'offset', NULL, 'size', NULL, 'content_type', NULL, 'checksum', NULL, 'inline', NULL) AS FILE) AS f FROM numbers("number"="3") ), file_scan AS ( SELECT id, f, CAST(ARRAY(CAST(f AS ${publicStruct})) AS ARRAY<FILE>) AS a, diff --git a/regression-test/suites/query_p0/cache/test_file_type_sql_cache.groovy b/regression-test/suites/query_p0/cache/test_file_type_sql_cache.groovy index c5e6a7a91b9..e29bd4cecba 100644 --- a/regression-test/suites/query_p0/cache/test_file_type_sql_cache.groovy +++ b/regression-test/suites/query_p0/cache/test_file_type_sql_cache.groovy @@ -42,22 +42,26 @@ suite("test_file_type_sql_cache") { sql "SET enable_sql_cache = true" sql "DROP TABLE IF EXISTS test_file_type_sql_cache_values" sql """ - CREATE TABLE test_file_type_sql_cache_values (id INT NOT NULL, j JSON) + CREATE TABLE test_file_type_sql_cache_values ( + id INT NOT NULL, uri STRING, file_offset BIGINT, file_size BIGINT, + content_type STRING, checksum STRING) DUPLICATE KEY(id) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES("replication_num"="1") """ sql """ INSERT INTO test_file_type_sql_cache_values VALUES - (1, JSON_PARSE('{"uri":"urn:cache:one","offset":2,"size":5, - "content_type":"text/plain","checksum":"CRC32:00000000","inline":null}')), - (2, NULL) + (1, 'urn:cache:one', 2, 5, 'text/plain', 'CRC32:00000000'), + (2, NULL, NULL, NULL, NULL, NULL) """ def statement = """ SELECT id, f, CAST(ARRAY(CAST(f AS ${publicStruct}), NULL) AS ARRAY<FILE>) AS files, CAST(NAMED_STRUCT('f', CAST(f AS ${publicStruct})) AS STRUCT<f:FILE>) AS holder, CAST(MAP('source', CAST(f AS ${publicStruct})) AS MAP<STRING,FILE>) AS lookup - FROM (SELECT id, CAST(j AS FILE) AS f FROM test_file_type_sql_cache_values) file_rows ORDER BY id + FROM (SELECT id, CAST(IF(uri IS NULL, NULL, + NAMED_STRUCT('uri', uri, 'offset', file_offset, 'size', file_size, + 'content_type', content_type, 'checksum', checksum, 'inline', NULL)) AS FILE) AS f + FROM test_file_type_sql_cache_values) file_rows ORDER BY id """ qt_uncached statement primeSqlCache(statement) @@ -65,7 +69,7 @@ suite("test_file_type_sql_cache") { sql """ INSERT INTO test_file_type_sql_cache_values VALUES - (3, JSON_PARSE('{"uri":"urn:cache:three","size":0,"offset":null,"content_type":null,"checksum":null,"inline":null}')) + (3, 'urn:cache:three', NULL, 0, NULL, NULL) """ assertFalse(hasSqlCache(statement), "A new table version must invalidate the cached FILE rows") qt_after_insert statement --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
