This is an automated email from the ASF dual-hosted git repository.
CalvinKirs pushed a change to branch 4.1-62196
in repository https://gitbox.apache.org/repos/asf/doris.git
from 7175b4d187f Merge remote-tracking branch 'rich/branch-4.1' into
4.1-62196
add 4c8235a2cbe branch-4.1: [improvement](paimon) bound memory for
synchronous JNI writes (#66810)
add 11286672f78 branch-4.1: [fix](be) Fix macOS build on branch-4.1
(#67011)
add 2eccc437cf0 branch-4.1:[fix](variant) Serve shredded Variant leaves
without rebuilding the root (#66941)
add a61493093f0 branch-4.1: [fix](paimon) Keep write cache coherent and
extend coverage (#67023)
add 383cf3395ae branch-4.1: [Fix](func) Keep multi-match regex cache owner
alive (#66909) (#67025)
add 5c9ac457a4d branch-4.1: [feature](catalog) Support Alibaba Cloud OSS
Tables REST catalog (#66823)
add f282a8994fd branch-4.1: [fix](memory) fix ThreadPoolExecutor leak in
PublishVersionDaemon #60720 (#67035)
add 325847f66f2 branch-4.1: [fix](streaming-job) Encode CDC stream load
records as UTF-8 #66771 (#67034)
add 461e06da53f [fix](exec) Return DATETIME as naive Arrow timestamp
(#67027)
add bde96dafa5e branch-4.1: [fix](workload policy)Remove set session
variable workload policy action #64856 (#67009)
add 9f1890df667 branch-4.1: [fix](cloud) Only mark prepare rowsets before
recycle (#65550) (#67014)
add bb1dea90724 branch-4.1: [fix](fd) Suppress invalid unique constraints
in scan FD derivation #66801 (#66912)
add 0e06fc1361b branch-4.1: [improvement](fe) Avoid under-estimating
not-in predicate row count #66632 (#66900)
add 0477eac5033 Merge remote-tracking branch 'rich/branch-4.1' into
4.1-62196
No new revisions were added by this update.
Summary of changes:
be/CMakeLists.txt | 17 +-
be/src/common/phdr_cache.h | 2 +
.../core/column/variant_v2/column_variant_v2.cpp | 24 +-
be/src/exec/connector/jni_connector.cpp | 4 +-
be/src/exec/operator/operator.cpp | 1 -
.../exec/operator/paimon_table_sink_operator.cpp | 56 +-
be/src/exec/operator/paimon_table_sink_operator.h | 44 +-
be/src/exec/operator/result_sink_operator.cpp | 6 +-
be/src/exec/operator/scan_operator.cpp | 12 +-
be/src/exec/operator/scan_operator.h | 2 +-
be/src/exec/pipeline/pipeline_fragment_context.cpp | 2 +-
.../writer/paimon/jni_paimon_write_backend.cpp | 256 ++--
.../sink/writer/paimon/jni_paimon_write_backend.h | 18 +-
.../writer/paimon/paimon_jni_memory_manager.cpp | 30 +-
.../sink/writer/paimon/paimon_jni_memory_manager.h | 19 +-
.../sink/writer/paimon/paimon_table_writer.cpp | 44 +-
.../exec/sink/writer/paimon/paimon_table_writer.h | 21 +-
.../exec/sink/writer/paimon/paimon_write_backend.h | 2 +-
.../exprs/aggregate/aggregate_function_java_udaf.h | 6 +-
.../function/functions_multi_string_search.cpp | 196 +--
..._search.cpp => functions_multi_string_search.h} | 137 +-
be/src/exprs/function/regexps.h | 12 +-
be/src/format/arrow/arrow_row_batch.cpp | 42 +-
be/src/format/arrow/arrow_row_batch.h | 7 +-
be/src/format_v2/column_mapper.cpp | 42 +-
be/src/format_v2/parquet/parquet_profile.cpp | 8 +
be/src/format_v2/parquet/parquet_profile.h | 5 +
be/src/format_v2/parquet/parquet_reader.cpp | 123 +-
be/src/format_v2/parquet/parquet_scan.cpp | 118 +-
be/src/format_v2/parquet/parquet_scan.h | 23 +-
.../parquet/reader/variant_column_reader.cpp | 486 +++++-
.../parquet/reader/variant_column_reader.h | 7 +
be/src/io/cache/block_file_cache_factory.cpp | 8 +
be/src/util/md5.cpp | 13 +-
be/test/core/column/column_variant_v2_test.cpp | 81 +
.../data_type_serde/data_type_serde_arrow_test.cpp | 75 +
.../exec/sink/paimon_jni_memory_manager_test.cpp | 64 +
.../functions_multi_string_search_test.cpp | 64 +
be/test/format_v2/column_mapper_test.cpp | 116 +-
be/test/format_v2/parquet/parquet_reader_test.cpp | 207 ++-
be/test/format_v2/parquet/parquet_scan_test.cpp | 2 +
.../format_v2/parquet/parquet_statistics_test.cpp | 4 +-
.../parquet/variant_column_reader_test.cpp | 1034 ++++++++++++-
cloud/src/meta-service/meta_service_job.cpp | 6 +-
cloud/src/recycler/recycler.cpp | 878 +++++++----
cloud/src/recycler/recycler.h | 54 +-
cloud/test/recycler_test.cpp | 1610 ++++++++++++++++++--
fe/be-java-extensions/paimon-connector/pom.xml | 9 +
.../doris/paimon/DorisMemorySegmentPool.java | 6 +-
.../doris/paimon/PaimonArrowBatchAdapter.java | 210 +++
.../apache/doris/paimon/PaimonArrowConverter.java | 426 ------
.../org/apache/doris/paimon/PaimonJniWriter.java | 154 +-
.../org/apache/doris/paimon/PaimonWriteSchema.java | 50 +-
.../doris/paimon/PaimonArrowBatchAdapterTest.java | 290 ++++
.../doris/paimon/PaimonArrowConverterTest.java | 244 ---
.../apache/doris/paimon/PaimonJniWriterTest.java | 24 +-
.../apache/doris/paimon/PaimonWriteSchemaTest.java | 49 +-
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 1 -
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 4 +-
.../main/java/org/apache/doris/catalog/Env.java | 5 +-
.../apache/doris/datasource/ExternalCatalog.java | 1 +
.../CatalogConnectivityTestCoordinator.java | 6 +-
.../IcebergRestConnectivityTester.java | 14 +-
.../doris/datasource/iceberg/IcebergUtils.java | 54 +
.../datasource/iceberg/source/IcebergScanNode.java | 1 +
.../metacache/paimon/PaimonTableLoader.java | 7 +-
.../datasource/paimon/PaimonExternalCatalog.java | 73 +-
.../datasource/paimon/PaimonExternalTable.java | 7 +-
.../metastore/AbstractIcebergProperties.java | 21 +-
.../property/metastore/IcebergRestProperties.java | 33 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 15 +-
.../doris/nereids/stats/FilterEstimation.java | 12 +-
.../plans/logical/LogicalCatalogRelation.java | 7 +-
.../trees/plans/logical/LogicalOlapScan.java | 28 +-
.../plans/physical/PhysicalCatalogRelation.java | 7 +-
.../org/apache/doris/planner/IcebergTableSink.java | 18 +-
.../workloadschedpolicy/WorkloadAction.java | 2 -
.../WorkloadActionSetSessionVar.java | 67 -
.../workloadschedpolicy/WorkloadActionType.java | 1 +
.../workloadschedpolicy/WorkloadSchedPolicy.java | 21 +-
.../WorkloadSchedPolicyMgr.java | 179 +--
.../service/arrowflight/FlightSqlSchemaHelper.java | 14 +-
.../IcebergRestConnectivityTesterTest.java | 57 +
.../doris/datasource/iceberg/IcebergUtilsTest.java | 30 +
.../paimon/PaimonExternalMetaCacheTest.java | 57 +
.../datasource/paimon/PaimonExternalTableTest.java | 17 +
.../metastore/IcebergRestPropertiesTest.java | 79 +
.../property/storage/StoragePropertiesTest.java | 18 +
.../apache/doris/nereids/properties/FdTest.java | 77 +
.../doris/nereids/stats/FilterEstimationTest.java | 58 +
.../apache/doris/planner/IcebergTableSinkTest.java | 48 +
.../WorkloadSchedPolicyMgrTest.java | 63 +-
fe/pom.xml | 15 +
.../cdcclient/service/PipelineCoordinator.java | 3 +-
.../data/arrow_flight_sql_p0/test_select.out | 6 +-
.../paimon/test_paimon_jdbc_catalog.out | 16 +
.../test_paimon_create_ddl_write_properties.out | 0
.../write}/test_paimon_write_append_only.out | 0
.../write}/test_paimon_write_bucket_modes.out | 7 +
.../test_paimon_write_changelog_producer.out | 0
.../paimon/write}/test_paimon_write_compaction.out | 0
.../write}/test_paimon_write_complex_types.out | 6 +-
.../write/test_paimon_write_deletion_vector.out | 38 +
.../paimon/write}/test_paimon_write_edge_cases.out | 0
.../write/test_paimon_write_external_paths.out | 77 +
.../paimon/write}/test_paimon_write_failures.out | 0
.../write}/test_paimon_write_merge_engine.out | 0
.../write/test_paimon_write_merge_semantics.out | 8 +
.../write/test_paimon_write_partition_delete.out | 20 +
.../paimon/write}/test_paimon_write_pk.out | 0
.../write}/test_paimon_write_row_level_dml.out | 0
.../test_paimon_write_row_tracking_evolution.out | 17 +
.../write}/test_paimon_write_schema_change.out | 0
.../write/test_paimon_write_sequence_group.out | 30 +
.../write/test_paimon_write_sequence_rowkind.out | 17 +
.../write}/test_paimon_write_transaction.out | 0
.../paimon/write}/test_paimon_write_types.out | 0
.../paimon/write}/test_paimon_write_variant.out | 0
.../write}/test_paimon_write_variant_dml.out | 0
.../write}/test_paimon_write_variant_errors.out | 0
.../write}/test_paimon_write_variant_nested.out | 0
.../write}/test_paimon_write_variant_shredding.out | 0
.../test_paimon_write_variant_table_modes.out | 0
.../test_workload_sched_policy.out | 12 +-
.../suites/arrow_flight_sql_p0/test_select.groovy | 65 +-
.../iceberg/test_iceberg_variant_read.groovy | 29 +
.../paimon/test_paimon_catalog_variant.groovy | 8 +-
.../paimon/test_paimon_jdbc_catalog.groovy | 135 +-
.../test_paimon_create_ddl_write_properties.groovy | 0
.../write}/test_paimon_write_append_only.groovy | 0
.../write}/test_paimon_write_bucket_modes.groovy | 70 +
.../test_paimon_write_changelog_producer.groovy | 0
.../write}/test_paimon_write_compaction.groovy | 0
.../write}/test_paimon_write_complex_types.groovy | 8 +-
.../write/test_paimon_write_deletion_vector.groovy | 194 +++
.../write}/test_paimon_write_edge_cases.groovy | 0
.../write/test_paimon_write_external_paths.groovy | 200 +++
.../write}/test_paimon_write_failures.groovy | 0
...paimon_write_key_dynamic_memory_negative.groovy | 0
.../write}/test_paimon_write_merge_engine.groovy | 0
.../write/test_paimon_write_merge_semantics.groovy | 221 +++
.../test_paimon_write_partition_delete.groovy | 150 ++
.../paimon/write}/test_paimon_write_pk.groovy | 0
.../write}/test_paimon_write_row_level_dml.groovy | 0
...test_paimon_write_row_tracking_evolution.groovy | 216 +++
.../write}/test_paimon_write_schema_change.groovy | 0
.../write/test_paimon_write_sequence_group.groovy | 227 +++
.../test_paimon_write_sequence_rowkind.groovy | 200 +++
.../write}/test_paimon_write_snapshot_refs.groovy | 0
.../write}/test_paimon_write_source_models.groovy | 0
.../test_paimon_write_thread_lifecycle.groovy | 0
.../write}/test_paimon_write_transaction.groovy | 0
.../paimon/write}/test_paimon_write_types.groovy | 0
.../paimon/write}/test_paimon_write_variant.groovy | 0
.../write}/test_paimon_write_variant_dml.groovy | 0
.../write}/test_paimon_write_variant_errors.groovy | 0
.../write}/test_paimon_write_variant_nested.groovy | 0
.../test_paimon_write_variant_shredding.groovy | 0
.../test_paimon_write_variant_table_modes.groovy | 0
.../test_workload_sched_policy.groovy | 89 +-
160 files changed, 8033 insertions(+), 2603 deletions(-)
copy be/src/exprs/function/{functions_multi_string_search.cpp =>
functions_multi_string_search.h} (63%)
create mode 100644 be/test/exec/sink/paimon_jni_memory_manager_test.cpp
create mode 100644
be/test/exprs/function/functions_multi_string_search_test.cpp
create mode 100644
fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonArrowBatchAdapter.java
delete mode 100644
fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonArrowConverter.java
create mode 100644
fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonArrowBatchAdapterTest.java
delete mode 100644
fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonArrowConverterTest.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/resource/workloadschedpolicy/WorkloadActionSetSessionVar.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/connectivity/IcebergRestConnectivityTesterTest.java
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_create_ddl_write_properties.out
(100%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_append_only.out (100%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_bucket_modes.out (88%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_changelog_producer.out (100%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_compaction.out (100%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_complex_types.out (96%)
create mode 100644
regression-test/data/external_table_p0/paimon/write/test_paimon_write_deletion_vector.out
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_edge_cases.out (100%)
create mode 100644
regression-test/data/external_table_p0/paimon/write/test_paimon_write_external_paths.out
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_failures.out (100%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_merge_engine.out (100%)
create mode 100644
regression-test/data/external_table_p0/paimon/write/test_paimon_write_merge_semantics.out
create mode 100644
regression-test/data/external_table_p0/paimon/write/test_paimon_write_partition_delete.out
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_pk.out (100%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_row_level_dml.out (100%)
create mode 100644
regression-test/data/external_table_p0/paimon/write/test_paimon_write_row_tracking_evolution.out
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_schema_change.out (100%)
create mode 100644
regression-test/data/external_table_p0/paimon/write/test_paimon_write_sequence_group.out
create mode 100644
regression-test/data/external_table_p0/paimon/write/test_paimon_write_sequence_rowkind.out
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_transaction.out (100%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_types.out (100%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_variant.out (100%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_variant_dml.out (100%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_variant_errors.out (100%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_variant_nested.out (100%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_variant_shredding.out (100%)
rename regression-test/data/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_variant_table_modes.out (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_create_ddl_write_properties.groovy
(100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_append_only.groovy (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_bucket_modes.groovy (87%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_changelog_producer.groovy
(100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_compaction.groovy (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_complex_types.groovy (96%)
create mode 100644
regression-test/suites/external_table_p0/paimon/write/test_paimon_write_deletion_vector.groovy
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_edge_cases.groovy (100%)
create mode 100644
regression-test/suites/external_table_p0/paimon/write/test_paimon_write_external_paths.groovy
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_failures.groovy (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_key_dynamic_memory_negative.groovy
(100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_merge_engine.groovy (100%)
create mode 100644
regression-test/suites/external_table_p0/paimon/write/test_paimon_write_merge_semantics.groovy
create mode 100644
regression-test/suites/external_table_p0/paimon/write/test_paimon_write_partition_delete.groovy
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_pk.groovy (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_row_level_dml.groovy (100%)
create mode 100644
regression-test/suites/external_table_p0/paimon/write/test_paimon_write_row_tracking_evolution.groovy
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_schema_change.groovy (100%)
create mode 100644
regression-test/suites/external_table_p0/paimon/write/test_paimon_write_sequence_group.groovy
create mode 100644
regression-test/suites/external_table_p0/paimon/write/test_paimon_write_sequence_rowkind.groovy
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_snapshot_refs.groovy (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_source_models.groovy (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_thread_lifecycle.groovy (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_transaction.groovy (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_types.groovy (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_variant.groovy (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_variant_dml.groovy (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_variant_errors.groovy (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_variant_nested.groovy (100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_variant_shredding.groovy
(100%)
rename regression-test/suites/{paimon_write =>
external_table_p0/paimon/write}/test_paimon_write_variant_table_modes.groovy
(100%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
