This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
from 33c460d7365 [fix](fe) Check automatic partition limit before creation
(#69128)
add cefe0230950 [fix](paimon) Truncate evolved timestamps to target
precision (#68161)
No new revisions were added by this update.
Summary of changes:
.licenserc.yaml | 4 +
be/src/format_v2/column_mapper.cpp | 28 +-
be/src/format_v2/column_mapper.h | 6 +
be/src/format_v2/expr/evolution_cast.cpp | 52 ++
be/src/format_v2/expr/evolution_cast.h | 10 +-
be/src/format_v2/table/paimon_reader.h | 1 +
be/src/format_v2/table_reader.h | 12 +-
be/test/format_v2/column_mapper_test.cpp | 103 +++
be/test/format_v2/expr/evolution_cast_test.cpp | 50 ++
.../docker-compose/iceberg/iceberg.yaml.tpl | 1 +
...data-0e4e0ae4-13ce-495b-a7a0-78a4cdcfe30d-0.orc | Bin 0 -> 401 bytes
...manifest-575ae3ae-ac95-41b9-ac9c-627b9e379898-0 | Bin 0 -> 2093 bytes
...est-list-85cb32d3-8ca6-423b-880d-3d94023779a6-0 | Bin 0 -> 884 bytes
...est-list-85cb32d3-8ca6-423b-880d-3d94023779a6-1 | Bin 0 -> 989 bytes
.../schema/schema-0 | 22 +
.../schema/schema-1 | 22 +
.../snapshot/LATEST | 0
.../snapshot/snapshot-1 | 20 +
...-40c7a472-ba63-4b7d-aa0f-b512cda13f0e-0.parquet | Bin 0 -> 535 bytes
...manifest-b482e455-6982-49c7-ac3c-bc618bee7bd5-0 | Bin 0 -> 2098 bytes
...est-list-d9fb2a7d-190b-4e78-822e-6b7f34f3451d-0 | Bin 0 -> 884 bytes
...est-list-d9fb2a7d-190b-4e78-822e-6b7f34f3451d-1 | Bin 0 -> 989 bytes
.../schema/schema-0 | 22 +
.../schema/schema-1 | 22 +
.../snapshot/LATEST | 0
.../snapshot/snapshot-1 | 20 +
...-aceafc60-6a85-4e6c-9303-3c4523025136-0.parquet | Bin 0 -> 1102 bytes
...manifest-43556a23-7d15-4a7d-b1fe-ef0f6b04615a-0 | Bin 0 -> 2113 bytes
...est-list-f3d303fe-7e62-4f63-9e17-b031105b985c-0 | Bin 0 -> 884 bytes
...est-list-f3d303fe-7e62-4f63-9e17-b031105b985c-1 | Bin 0 -> 983 bytes
.../schema/schema-0 | 30 +
.../schema/schema-1 | 30 +
.../snapshot/LATEST | 0
.../snapshot/snapshot-1 | 20 +
...-df5aad5a-827a-4bb9-927d-783200ec15b0-0.parquet | Bin 0 -> 897 bytes
...manifest-3940a649-76fc-4fb4-ac46-a4c759baba76-0 | Bin 0 -> 2090 bytes
...est-list-aef4d9a2-7c93-42fb-8b07-0984316518c7-0 | Bin 0 -> 884 bytes
...est-list-aef4d9a2-7c93-42fb-8b07-0984316518c7-1 | Bin 0 -> 984 bytes
.../schema/schema-0 | 36 +
.../schema/schema-1 | 36 +
.../schema/schema-2 | 36 +
.../snapshot/LATEST | 0
.../snapshot/snapshot-1 | 20 +
...-2eb67d55-56cd-44e4-8b28-4884fa0398c1-0.parquet | Bin 0 -> 1266 bytes
...manifest-26ef652b-eb81-4004-88d7-7af675defd83-0 | Bin 0 -> 2138 bytes
...est-list-00639421-36ab-4396-8c9c-ce36bd61ac75-0 | Bin 0 -> 1006 bytes
...est-list-00639421-36ab-4396-8c9c-ce36bd61ac75-1 | Bin 0 -> 1109 bytes
.../schema/schema-0 | 32 +
.../schema/schema-1 | 32 +
.../snapshot/EARLIEST | 0
.../snapshot/LATEST | 0
.../snapshot/snapshot-1 | 16 +
...data-e310e0fb-3049-4d92-a9f5-4d021d32327c-0.orc | Bin 0 -> 538 bytes
...manifest-2223cd32-36ff-4f7e-92ea-b60bfa02f487-0 | Bin 0 -> 2115 bytes
...est-list-ee429867-e956-487a-a21c-1a960482b5bd-0 | Bin 0 -> 884 bytes
...est-list-ee429867-e956-487a-a21c-1a960482b5bd-1 | Bin 0 -> 984 bytes
.../schema/schema-0 | 27 +
.../schema/schema-1 | 27 +
.../schema/schema-2 | 27 +
.../snapshot/LATEST | 0
.../snapshot/snapshot-1 | 20 +
...data-0449dfa1-48bb-41be-813f-4195bf9bee95-0.orc | Bin 0 -> 406 bytes
...manifest-1ec1b6cb-4403-41d6-81a6-9b7fd3f5273f-0 | Bin 0 -> 2108 bytes
...est-list-f72e0a0b-9a78-4850-a84a-8c35d28e3b33-0 | Bin 0 -> 884 bytes
...est-list-f72e0a0b-9a78-4850-a84a-8c35d28e3b33-1 | Bin 0 -> 989 bytes
.../schema/schema-0 | 22 +
.../schema/schema-1 | 22 +
.../snapshot/LATEST | 0
.../snapshot/snapshot-1 | 20 +
...-501848c5-b1cf-4c3e-af8f-103b7d377827-0.parquet | Bin 0 -> 561 bytes
...manifest-05e65016-fbe3-40e3-a0d9-2dcc01cb4c2b-0 | Bin 0 -> 2113 bytes
...est-list-d8d74639-df69-453c-a7d8-8d66335f7718-0 | Bin 0 -> 884 bytes
...est-list-d8d74639-df69-453c-a7d8-8d66335f7718-1 | Bin 0 -> 983 bytes
.../schema/schema-0 | 22 +
.../schema/schema-1 | 22 +
.../snapshot/LATEST | 0
.../snapshot/snapshot-1 | 20 +
.../org/apache/doris/paimon/PaimonColumnValue.java | 40 +-
.../org/apache/doris/paimon/PaimonJniScanner.java | 214 +++++-
.../doris/paimon/PaimonTimestampQueryAuth.java | 204 ++++++
.../apache/doris/paimon/PaimonColumnValueTest.java | 79 ++
.../apache/doris/paimon/PaimonJniScannerTest.java | 199 +++++
.../paimon/PaimonTimestampRepairReaderTest.java | 807 +++++++++++++++++++++
.../connector/paimon/PaimonPredicateConverter.java | 32 +-
.../connector/paimon/PaimonScanPlanProvider.java | 251 ++++++-
.../paimon/PaimonLegacyOrcTimestampTest.java | 10 +-
.../paimon/PaimonPredicateConverterTest.java | 45 ++
.../paimon/PaimonScanPlanProviderTest.java | 421 +++++++++++
.../paimon/paimon_timestamp_types.out | 139 ++++
.../paimon/paimon_timestamp_types.groovy | 149 +++-
90 files changed, 3451 insertions(+), 51 deletions(-)
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/bucket-0/data-0e4e0ae4-13ce-495b-a7a0-78a4cdcfe30d-0.orc
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/manifest/manifest-575ae3ae-ac95-41b9-ac9c-627b9e379898-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/manifest/manifest-list-85cb32d3-8ca6-423b-880d-3d94023779a6-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/manifest/manifest-list-85cb32d3-8ca6-423b-880d-3d94023779a6-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/schema/schema-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/schema/schema-1
copy
docker/thirdparties/docker-compose/{hive/scripts/paimon1/db1.db/all_table_with_parquet
=>
iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc}/snapshot/LATEST
(100%)
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_orc/snapshot/snapshot-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/bucket-0/data-40c7a472-ba63-4b7d-aa0f-b512cda13f0e-0.parquet
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/manifest/manifest-b482e455-6982-49c7-ac3c-bc618bee7bd5-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/manifest/manifest-list-d9fb2a7d-190b-4e78-822e-6b7f34f3451d-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/manifest/manifest-list-d9fb2a7d-190b-4e78-822e-6b7f34f3451d-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/schema/schema-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/schema/schema-1
copy
docker/thirdparties/docker-compose/{hive/scripts/paimon1/db1.db/all_table_with_parquet
=>
iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet}/snapshot/LATEST
(100%)
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_leading_zero_parquet/snapshot/snapshot-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/bucket-0/data-aceafc60-6a85-4e6c-9303-3c4523025136-0.parquet
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/manifest/manifest-43556a23-7d15-4a7d-b1fe-ef0f6b04615a-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/manifest/manifest-list-f3d303fe-7e62-4f63-9e17-b031105b985c-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/manifest/manifest-list-f3d303fe-7e62-4f63-9e17-b031105b985c-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/schema/schema-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/schema/schema-1
copy
docker/thirdparties/docker-compose/{hive/scripts/paimon1/db1.db/all_table_with_parquet
=>
iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet}/snapshot/LATEST
(100%)
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_map_key_parquet/snapshot/snapshot-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/bucket-0/data-df5aad5a-827a-4bb9-927d-783200ec15b0-0.parquet
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/manifest/manifest-3940a649-76fc-4fb4-ac46-a4c759baba76-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/manifest/manifest-list-aef4d9a2-7c93-42fb-8b07-0984316518c7-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/manifest/manifest-list-aef4d9a2-7c93-42fb-8b07-0984316518c7-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/schema/schema-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/schema/schema-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/schema/schema-2
copy
docker/thirdparties/docker-compose/{hive/scripts/paimon1/db1.db/all_table_with_parquet
=>
iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet}/snapshot/LATEST
(100%)
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/doris_29023_timestamp_nested_parquet/snapshot/snapshot-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/bucket-0/data-2eb67d55-56cd-44e4-8b28-4884fa0398c1-0.parquet
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/manifest/manifest-26ef652b-eb81-4004-88d7-7af675defd83-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/manifest/manifest-list-00639421-36ab-4396-8c9c-ce36bd61ac75-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/manifest/manifest-list-00639421-36ab-4396-8c9c-ce36bd61ac75-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/schema/schema-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/schema/schema-1
copy
docker/thirdparties/docker-compose/{hive/scripts/paimon1/db1.db/append_table =>
iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet}/snapshot/EARLIEST
(100%)
copy
docker/thirdparties/docker-compose/{hive/scripts/paimon1/db1.db/all_table_with_parquet
=>
iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet}/snapshot/LATEST
(100%)
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_cross_family_evolution_parquet/snapshot/snapshot-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/bucket-0/data-e310e0fb-3049-4d92-a9f5-4d021d32327c-0.orc
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/manifest/manifest-2223cd32-36ff-4f7e-92ea-b60bfa02f487-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/manifest/manifest-list-ee429867-e956-487a-a21c-1a960482b5bd-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/manifest/manifest-list-ee429867-e956-487a-a21c-1a960482b5bd-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/schema/schema-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/schema/schema-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/schema/schema-2
copy
docker/thirdparties/docker-compose/{hive/scripts/paimon1/db1.db/all_table_with_parquet
=>
iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc}/snapshot/LATEST
(100%)
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_nanosecond_evolution_orc/snapshot/snapshot-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/bucket-0/data-0449dfa1-48bb-41be-813f-4195bf9bee95-0.orc
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/manifest/manifest-1ec1b6cb-4403-41d6-81a6-9b7fd3f5273f-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/manifest/manifest-list-f72e0a0b-9a78-4850-a84a-8c35d28e3b33-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/manifest/manifest-list-f72e0a0b-9a78-4850-a84a-8c35d28e3b33-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/schema/schema-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/schema/schema-1
copy
docker/thirdparties/docker-compose/{hive/scripts/paimon1/db1.db/all_table_with_parquet
=>
iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc}/snapshot/LATEST
(100%)
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_orc/snapshot/snapshot-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/bucket-0/data-501848c5-b1cf-4c3e-af8f-103b7d377827-0.parquet
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/manifest/manifest-05e65016-fbe3-40e3-a0d9-2dcc01cb4c2b-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/manifest/manifest-list-d8d74639-df69-453c-a7d8-8d66335f7718-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/manifest/manifest-list-d8d74639-df69-453c-a7d8-8d66335f7718-1
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/schema/schema-0
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/schema/schema-1
copy
docker/thirdparties/docker-compose/{hive/scripts/paimon1/db1.db/all_table_with_parquet
=>
iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet}/snapshot/LATEST
(100%)
create mode 100644
docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/paimon/flink_paimon.db/timestamp_precision_evolution_parquet/snapshot/snapshot-1
create mode 100644
fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonTimestampQueryAuth.java
create mode 100644
fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonTimestampRepairReaderTest.java
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
