This is an automated email from the ASF dual-hosted git repository.
zhangstar333 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
from 874c1cf4adf [fix](mtmv)fix MTMV staying in sync after base table
recreate column (#58087)
add c1eaede1260 [feature](catalog) support varbinary type mapping in
hive/iceberg/paimon table (#57821)
No new revisions were added by this update.
Summary of changes:
be/src/util/arrow/row_batch.cpp | 4 +
be/src/vec/columns/column_varbinary.cpp | 60 ++++
be/src/vec/columns/column_varbinary.h | 20 +-
.../data_types/serde/data_type_varbinary_serde.cpp | 65 ++++
.../data_types/serde/data_type_varbinary_serde.h | 24 +-
be/src/vec/exec/format/column_type_convert.cpp | 9 +-
be/src/vec/exec/format/column_type_convert.h | 170 +++++++---
be/src/vec/exec/format/orc/vorc_reader.cpp | 14 +-
.../exec/format/parquet/parquet_column_convert.cpp | 16 +-
.../exec/format/parquet/parquet_column_convert.h | 43 +++
.../vec/exec/format/parquet/parquet_thrift_util.h | 4 +-
be/src/vec/exec/format/parquet/schema_desc.cpp | 23 +-
be/src/vec/exec/format/parquet/schema_desc.h | 3 +
.../exec/format/parquet/vparquet_file_metadata.cpp | 3 +-
.../exec/format/parquet/vparquet_file_metadata.h | 2 +-
be/src/vec/exec/format/parquet/vparquet_reader.cpp | 12 +-
be/src/vec/functions/cast/function_cast.cpp | 11 +
be/src/vec/io/io_helper.h | 32 +-
be/src/vec/runtime/vorc_transformer.cpp | 5 +
.../test_data/parquet_scanner/dict-decoder.txt | 32 +-
.../test_data/parquet_scanner/test_uuid.parquet | Bin 0 -> 509 bytes
.../test_data/parquet_scanner/type-decoder.txt | 28 +-
be/test/vec/columns/column_varbinary_test.cpp | 359 ++++++++++++++++++++-
.../serde/data_type_serde_varbinary_test.cpp | 197 ++++++++++-
.../vec/exec/format/parquet/parquet_expr_test.cpp | 2 +-
.../exec/format/parquet/parquet_reader_test.cpp | 284 ++++++++++++++++
.../exec/format/parquet/parquet_thrift_test.cpp | 9 +-
.../scripts/create_preinstalled_scripts/run82.hql | 6 -
.../scripts/create_preinstalled_scripts/run83.hql | 144 +++++++++
...-01367323-fe57-4cf2-8d63-658136eef42a-0.parquet | Bin 0 -> 1649 bytes
...-6a7f405a-2ffe-4c8d-97b8-5a9f69a497c6-0.parquet | Bin 0 -> 1419 bytes
...-f52a6587-0fcb-4d15-895d-6be35077eb8d-0.parquet | Bin 0 -> 1649 bytes
.../index-86b1c133-2b65-48d2-97b5-1c311f4c32b3-0 | 1 +
.../index-b00cbd60-d07f-462c-9ddc-effcf2ba6377-0 | 1 +
.../index-f6d8156c-7d1e-4643-beb4-f1040d4a8b71-0} | 0
...manifest-044381f8-72c7-4bbe-b3e9-d4829b7dc2c7-0 | Bin 0 -> 778 bytes
...manifest-a5543fe0-8526-4707-83bb-4a46bae33d32-0 | Bin 0 -> 778 bytes
...manifest-e5878ae9-97ff-4d9d-9e16-3601ef09cc9b-0 | Bin 0 -> 778 bytes
...manifest-0f70c85a-f753-4ee3-b9dd-e81dd513ee3c-0 | Bin 0 -> 1782 bytes
...manifest-84997273-d8aa-452d-95b6-42c88639a8ce-0 | Bin 0 -> 1800 bytes
...manifest-f4bb99cc-ede0-4196-8a93-3685e67fa218-0 | Bin 0 -> 1800 bytes
...est-list-21d352ee-7901-4ca3-af30-dd0e43bedcb9-0 | Bin 0 -> 770 bytes
...est-list-21d352ee-7901-4ca3-af30-dd0e43bedcb9-1 | Bin 0 -> 743 bytes
...est-list-3dfd6008-44c0-4227-8a3f-cf4c96419518-0 | Bin 0 -> 743 bytes
...est-list-3dfd6008-44c0-4227-8a3f-cf4c96419518-1 | Bin 0 -> 743 bytes
...est-list-9a34dd3e-af29-4b67-81c4-db2d00637a41-0 | Bin 0 -> 646 bytes
...est-list-9a34dd3e-af29-4b67-81c4-db2d00637a41-1 | Bin 0 -> 743 bytes
.../paimon1/db1.db/binary_demo3/schema/schema-0 | 31 ++
.../snapshot/EARLIEST | 0
.../{complex_all => binary_demo3}/snapshot/LATEST | 0
.../db1.db/binary_demo3/snapshot/snapshot-1 | 18 ++
.../db1.db/binary_demo3/snapshot/snapshot-2 | 18 ++
.../db1.db/binary_demo3/snapshot/snapshot-3 | 18 ++
...-5e407798-98e4-47e9-b884-ce798c662f92-0.parquet | Bin 0 -> 5936 bytes
.../index-5913b702-bb8d-4e33-b764-78c42470e82b-0} | 0
...manifest-3369567e-e644-447e-8c91-2feb5ef9079b-0 | Bin 0 -> 778 bytes
...manifest-8b25f42e-072c-4ccb-8ebf-e5faf2c65f03-0 | Bin 0 -> 1800 bytes
...est-list-1a56808b-dfe6-4f71-a3c3-49d8b428ca8d-0 | Bin 0 -> 646 bytes
...est-list-1a56808b-dfe6-4f71-a3c3-49d8b428ca8d-1 | Bin 0 -> 743 bytes
.../db1.db/binary_size_test/schema/schema-0 | 31 ++
.../snapshot/EARLIEST | 0
.../snapshot/LATEST | 0
.../db1.db/binary_size_test/snapshot/snapshot-1 | 18 ++
.../create_preinstalled_scripts/iceberg/run22.sql | 91 ++++++
.../java/org/apache/doris/catalog/ScalarType.java | 2 +
.../doris/common/util/FileFormatConstants.java | 1 +
.../apache/doris/datasource/CatalogProperty.java | 18 ++
.../apache/doris/datasource/ExternalCatalog.java | 10 +-
.../apache/doris/datasource/FileQueryScanNode.java | 26 ++
.../doris/datasource/hive/HMSExternalTable.java | 3 +-
.../datasource/hive/HiveMetaStoreClientHelper.java | 20 +-
.../doris/datasource/iceberg/IcebergUtils.java | 36 ++-
.../datasource/paimon/PaimonExternalTable.java | 3 +-
.../apache/doris/datasource/paimon/PaimonUtil.java | 50 +--
.../datasource/paimon/source/PaimonScanNode.java | 3 +-
.../property/fileformat/FileFormatProperties.java | 3 +
.../nereids/rules/expression/check/CheckCast.java | 2 +
.../rules/expression/rules/SimplifyCastRule.java | 16 +
.../expressions/literal/VarBinaryLiteral.java | 4 +
.../org/apache/doris/nereids/types/DataType.java | 2 +-
.../apache/doris/nereids/types/VarBinaryType.java | 4 +-
.../ExternalFileTableValuedFunction.java | 8 +
.../tablefunction/IcebergTableValuedFunction.java | 2 +-
.../tablefunction/PaimonTableValuedFunction.java | 2 +-
.../doris/datasource/paimon/PaimonUtilTest.java | 4 +-
gensrc/thrift/PlanNodes.thrift | 2 +
.../hive/test_hive_varbinary_type.out | 261 +++++++++++++++
.../iceberg/test_iceberg_varbinary.out | 85 +++++
.../paimon/test_paimon_catalog_varbinary.out | 29 ++
.../tvf/test_hdfs_parquet_group6.out | 8 +-
.../binary_functions/test_binary_hex_function.out | 28 ++
.../hive/test_hive_varbinary_type.groovy | 92 ++++++
.../iceberg/test_iceberg_varbinary.groovy | 152 +++++++++
.../paimon/test_paimon_catalog_varbinary.groovy | 97 ++++++
.../test_binary_hex_function.groovy | 4 +
95 files changed, 2571 insertions(+), 214 deletions(-)
create mode 100644 be/test/exec/test_data/parquet_scanner/test_uuid.parquet
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/create_preinstalled_scripts/run83.hql
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/bucket-0/data-01367323-fe57-4cf2-8d63-658136eef42a-0.parquet
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/bucket-0/data-6a7f405a-2ffe-4c8d-97b8-5a9f69a497c6-0.parquet
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/bucket-0/data-f52a6587-0fcb-4d15-895d-6be35077eb8d-0.parquet
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/index/index-86b1c133-2b65-48d2-97b5-1c311f4c32b3-0
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/index/index-b00cbd60-d07f-462c-9ddc-effcf2ba6377-0
copy
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/{auto_bucket/index/index-da673059-ffd4-41e6-9525-20581001517c-0
=> binary_demo3/index/index-f6d8156c-7d1e-4643-beb4-f1040d4a8b71-0} (100%)
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/manifest/index-manifest-044381f8-72c7-4bbe-b3e9-d4829b7dc2c7-0
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/manifest/index-manifest-a5543fe0-8526-4707-83bb-4a46bae33d32-0
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/manifest/index-manifest-e5878ae9-97ff-4d9d-9e16-3601ef09cc9b-0
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/manifest/manifest-0f70c85a-f753-4ee3-b9dd-e81dd513ee3c-0
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/manifest/manifest-84997273-d8aa-452d-95b6-42c88639a8ce-0
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/manifest/manifest-f4bb99cc-ede0-4196-8a93-3685e67fa218-0
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/manifest/manifest-list-21d352ee-7901-4ca3-af30-dd0e43bedcb9-0
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/manifest/manifest-list-21d352ee-7901-4ca3-af30-dd0e43bedcb9-1
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/manifest/manifest-list-3dfd6008-44c0-4227-8a3f-cf4c96419518-0
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/manifest/manifest-list-3dfd6008-44c0-4227-8a3f-cf4c96419518-1
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/manifest/manifest-list-9a34dd3e-af29-4b67-81c4-db2d00637a41-0
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/manifest/manifest-list-9a34dd3e-af29-4b67-81c4-db2d00637a41-1
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/schema/schema-0
copy
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/{append_table =>
binary_demo3}/snapshot/EARLIEST (100%)
copy
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/{complex_all =>
binary_demo3}/snapshot/LATEST (100%)
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/snapshot/snapshot-1
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/snapshot/snapshot-2
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_demo3/snapshot/snapshot-3
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_size_test/bucket-0/data-5e407798-98e4-47e9-b884-ce798c662f92-0.parquet
copy
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/{auto_bucket/index/index-da673059-ffd4-41e6-9525-20581001517c-0
=> binary_size_test/index/index-5913b702-bb8d-4e33-b764-78c42470e82b-0} (100%)
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_size_test/manifest/index-manifest-3369567e-e644-447e-8c91-2feb5ef9079b-0
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_size_test/manifest/manifest-8b25f42e-072c-4ccb-8ebf-e5faf2c65f03-0
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_size_test/manifest/manifest-list-1a56808b-dfe6-4f71-a3c3-49d8b428ca8d-0
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_size_test/manifest/manifest-list-1a56808b-dfe6-4f71-a3c3-49d8b428ca8d-1
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_size_test/schema/schema-0
copy
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/{append_table =>
binary_size_test}/snapshot/EARLIEST (100%)
copy
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/{all_table_with_parquet
=> binary_size_test}/snapshot/LATEST (100%)
create mode 100644
docker/thirdparties/docker-compose/hive/scripts/paimon1/db1.db/binary_size_test/snapshot/snapshot-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/create_preinstalled_scripts/iceberg/run22.sql
create mode 100644
regression-test/data/external_table_p0/hive/test_hive_varbinary_type.out
create mode 100644
regression-test/data/external_table_p0/iceberg/test_iceberg_varbinary.out
create mode 100644
regression-test/data/external_table_p0/paimon/test_paimon_catalog_varbinary.out
create mode 100644
regression-test/suites/external_table_p0/hive/test_hive_varbinary_type.groovy
create mode 100644
regression-test/suites/external_table_p0/iceberg/test_iceberg_varbinary.groovy
create mode 100644
regression-test/suites/external_table_p0/paimon/test_paimon_catalog_varbinary.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
