This is an automated email from the ASF dual-hosted git repository.
morningman pushed a change to branch branch-catalog-spi
in repository https://gitbox.apache.org/repos/asf/doris.git
omit e0c392b9be0 [fix](build) fix parallel -T shade race in
fe-connector-paimon-hive-shade
omit 83bc9259aa3 [fix](catalog) reconcile upstream #65365 with connector
SPI after rebase to 4f48ccfaa7d
omit 22af8613ef7 [doc](catalog) P6.6-C6 #63068 HANDOFF: record
cache-leakage data-flow test + Task 0 retained-base verify
omit 5c2263694c1 [feat](catalog) P6.6-C6 #63068 FE: re-migrate
ExternalDatabaseSessionContextTest (cross-user cache-leakage data-flow)
omit 7b26b49dd6b [doc](catalog) P6.6-C6 #63068 OIDC session migration:
design + research notes + HANDOFF (Tasks 0-7 DONE)
omit 2f9b1c9caa3 [feat](catalog) P6.6-C6 #63068 iceberg: per-user REST
session routing (metadata + scan/write/procedure)
omit 4ffdc391aec [feat](catalog) P6.6-C6 #63068 FE: capability-gated
credential injection + per-user cache bypass
omit 1ea883b711d [feat](catalog) P6.6-C6 #63068 SPI: neutral
delegated-credential DTO + user-session capability
omit 3670071e986 [fix](catalog) tier3 #65094: preserve external-table
top-level column-name case on the read path
omit 97be32c43d6 [fix](catalog) port upstream #65332/#65094 datasource
fixes into SPI connectors
omit 02e7bb44b49 [fix](catalog) P4 maxcompute: drop dangling legacy
MaxComputeExternalTableTest after rebase
omit 4363a8519f6 [fix](catalog) jdbc: keep driver classloaders alive per
URL to stop Metaspace leak
omit ec3a0cd55f4 [refactor](catalog) P6 iceberg: migrate to catalog SPI +
cutover + remove legacy fe-core subsystem (#64688)
omit c6a1f94eeec [refactor](catalog) P3b: consolidate kerberos
authentication into fe-kerberos (#64655)
omit 6cc203d1e0d [refactor](catalog) P5 paimon: remove legacy subsystem
from fe-core + make fe-core paimon-SDK-free (T29) (#64653)
omit 1baf73fe026 [refactor](catalog) P5 paimon: migrate to catalog SPI +
cutover (#64446) (#64446)
omit 1b8b7d63840 update P5 handoff and fix compile issue
omit 19b6d293834 [refactor](connector) P4 maxcompute: remove legacy
subsystem from fe-core + make fe-core odps-free (T07-T09) (#64300)
omit 133708b4698 [feat](connector) P3 hudi connector hardening + test
baseline + dispatch design (hybrid, T02-T08) (#64143)
omit e0c1f8fbd9b [feat](connector) P2 migrate trino-connector to catalog
SPI (T01-T13) (#64096)
omit 3a2d2aa80b0 [P1-T03-T05] route plugin-driven scans first in nereids
translator (#63641)
omit 21e0f021e6c [feat](connector) P0 SPI baseline + DDL/Partition + import
gate (T03-T27) (#63582)
omit 9345d787d23 [doc](connector) add project tracking system for catalog
SPI migration
add 17f0459fc23 [fe](pom) use java 17 not 9 (#65517)
add 893108f7c3e [fix](transaction) Fail fast when a backend restarts
during insert (#65525)
add 2eb014caf0b [improvement](be) Skip reading zonemap always-true columns
for count on index (#65242)
add c7599574472 [improve](streaming-job) Support MySQL ADD DROP schema
changes (#65325)
add 33a9a70afce [test](regression) Add warmup mixed-mode coverage and
debug observability (#65153)
add 4809030bd52 [improvement](be) Optimize hash join probe side output
(#65300)
add df4ba0f7c69 [test](regression) Remove test_decimalv2_common case
(#65540)
add 5f6e9b1c273 [Enhancement] (binlog<row>) move write lsn on sink, ensure
lsn same on all replica (#64133)
add 5c8faa4372e [be](chore) add some debug info to be logs (#65452)
add 1e47dc9b7c2 [fix](pipeline) prevent ColumnString64 across operator
boundaries (#65518)
add 052880662b0 [fix](agg) Fix distinct aggregate argument grouping and
strategy selection (#65368)
add 78836f200a0 [fix](regression) Wait for cloud tablet meta sync in
compaction test (#65184)
add b24f4beccc9 [fix](be) Disable DSL cache for score queries (#65436)
add f42bb06db13 [bugfix](memory) should force refresh workload group's
config when memlimit changed (#65542)
add b17e271f329 [fix](join) Handle oversized nested loop join blocks
(#65327)
add 43a31acc4b4 [fix](be) Validate Arrow input buffers before column
conversion (#64796)
add a1e076e8d69 [fix](compaction) Avoid repeatedly compacting large
cumulative rowset (#64954)
add e88dc45b2b6 [fix](be) Fix FileScannerV2 runtime filter profile and
page cache (#65580)
add f5c30d7d904 [feature](file-cache) Disable cache writes after remote
scan threshold (#65058)
add 0814e49bea7 [feature](iceberg) Support Iceberg position deletes system
table (#65135)
add f2673478bde [fix](filecache) skip redundant ttl scans for non-TTL
tablets (#65434)
add 7f302a19a1d [fix](fe) Reuse Ranger Doris access controller (#65570)
add 30a44d61a8a [improvement](be) Optimize filtered Parquet scans and
condition cache (#65602)
add 82dc967dc2c [doc](connector) add project tracking system for catalog
SPI migration
add 637c0cf6cd2 [feat](connector) P0 SPI baseline + DDL/Partition + import
gate (T03-T27) (#63582)
add 323cf213b6b [P1-T03-T05] route plugin-driven scans first in nereids
translator (#63641)
add 20fbd1916d2 [feat](connector) P2 migrate trino-connector to catalog
SPI (T01-T13) (#64096)
add 6acfc131e42 [feat](connector) P3 hudi connector hardening + test
baseline + dispatch design (hybrid, T02-T08) (#64143)
add 85a71916eea [refactor](connector) P4 maxcompute: remove legacy
subsystem from fe-core + make fe-core odps-free (T07-T09) (#64300)
add 8b4394a30da update P5 handoff and fix compile issue
add c276e955683 [refactor](catalog) P5 paimon: migrate to catalog SPI +
cutover (#64446) (#64446)
add f52c9672018 [refactor](catalog) P5 paimon: remove legacy subsystem
from fe-core + make fe-core paimon-SDK-free (T29) (#64653)
add 60f99269991 [refactor](catalog) P3b: consolidate kerberos
authentication into fe-kerberos (#64655)
add 442a1081e6d [refactor](catalog) P6 iceberg: migrate to catalog SPI +
cutover + remove legacy fe-core subsystem (#64688)
add 743e5f101a6 [fix](catalog) jdbc: keep driver classloaders alive per
URL to stop Metaspace leak
add 01d23afd17c [fix](catalog) P4 maxcompute: drop dangling legacy
MaxComputeExternalTableTest after rebase
add 2ac353fc67f [fix](catalog) port upstream #65332/#65094 datasource
fixes into SPI connectors
add 441917635ea [fix](catalog) tier3 #65094: preserve external-table
top-level column-name case on the read path
add 6f085e4b4ff [feat](catalog) P6.6-C6 #63068 SPI: neutral
delegated-credential DTO + user-session capability
add cab9aab728c [feat](catalog) P6.6-C6 #63068 FE: capability-gated
credential injection + per-user cache bypass
add d6adf3b7f17 [feat](catalog) P6.6-C6 #63068 iceberg: per-user REST
session routing (metadata + scan/write/procedure)
add 7d1d03aabbb [doc](catalog) P6.6-C6 #63068 OIDC session migration:
design + research notes + HANDOFF (Tasks 0-7 DONE)
add e7e73b34094 [feat](catalog) P6.6-C6 #63068 FE: re-migrate
ExternalDatabaseSessionContextTest (cross-user cache-leakage data-flow)
add a319b978a81 [doc](catalog) P6.6-C6 #63068 HANDOFF: record
cache-leakage data-flow test + Task 0 retained-base verify
add b8c8accadfd [fix](catalog) reconcile upstream #65365 with connector
SPI after rebase to 4f48ccfaa7d
add c215c4c77c5 [fix](build) fix parallel -T shade race in
fe-connector-paimon-hive-shade
add fb67a1a7f51 [doc](catalog) HANDOFF: rebase to upstream 30a44d61a8a +
register $position_deletes connector gap as flip blocker
add af0f0582bdf [doc](catalog) P6.6 $position_deletes connector port:
research notes + design (T0 evidence in, ready to implement)
add 95125c573b0 [doc](catalog) HANDOFF: point next step at the
$position_deletes port design (T0 evidence in)
add 2e49827ecdd [feat](catalog) P6.6: support the iceberg
$position_deletes system table on the connector SPI
add c0865b021b0 [doc](catalog) HANDOFF: $position_deletes port done; e2e
is the only remaining flip gate
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (e0c392b9be0)
\
N -- N -- N refs/heads/branch-catalog-spi (c0865b021b0)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
be/benchmark/benchmark_arrow_validation.hpp | 382 +
be/benchmark/benchmark_main.cpp | 1 +
.../cloud/cloud_cumulative_compaction_policy.cpp | 17 +-
be/src/cloud/cloud_delta_writer.cpp | 6 +-
be/src/cloud/cloud_delta_writer.h | 2 +-
be/src/cloud/cloud_tablets_channel.cpp | 8 +-
be/src/cloud/pb_convert.cpp | 44 +-
be/src/common/config.cpp | 5 +
be/src/common/config.h | 4 +
be/src/core/block/block.cpp | 14 +
be/src/core/block/block.h | 2 +
be/src/core/column/column.h | 15 +
be/src/core/data_type_serde/arrow_validation.h | 320 +
.../core/data_type_serde/data_type_array_serde.cpp | 5 +
.../data_type_date_or_datetime_serde.cpp | 5 +
.../data_type_serde/data_type_datetimev2_serde.cpp | 5 +
.../data_type_serde/data_type_datev2_serde.cpp | 5 +
.../data_type_serde/data_type_decimal_serde.cpp | 5 +
.../core/data_type_serde/data_type_ipv4_serde.cpp | 5 +
.../core/data_type_serde/data_type_ipv6_serde.cpp | 5 +
.../core/data_type_serde/data_type_jsonb_serde.cpp | 5 +
.../core/data_type_serde/data_type_map_serde.cpp | 5 +
.../data_type_serde/data_type_nullable_serde.cpp | 6 +
.../data_type_serde/data_type_number_serde.cpp | 26 +-
.../data_type_serde/data_type_string_serde.cpp | 39 +-
.../data_type_serde/data_type_struct_serde.cpp | 5 +
.../exec/operator/join/process_hash_table_probe.h | 4 +-
.../operator/join/process_hash_table_probe_impl.h | 74 +-
.../operator/nested_loop_join_probe_operator.cpp | 27 +-
be/src/exec/operator/operator.h | 2 +
be/src/exec/scan/file_scan_io_context.h | 44 +
be/src/exec/scan/file_scanner.cpp | 64 +-
be/src/exec/scan/file_scanner.h | 4 +-
be/src/exec/scan/file_scanner_v2.cpp | 22 +-
be/src/exec/scan/olap_scanner.cpp | 46 +-
be/src/exec/scan/olap_scanner.h | 9 +
be/src/exec/scan/parallel_scanner_builder.cpp | 21 +-
be/src/exec/scan/scanner.cpp | 34 +
be/src/exec/scan/scanner.h | 12 +
be/src/exec/sink/writer/vtablet_writer.cpp | 42 +-
be/src/exec/sink/writer/vtablet_writer.h | 4 +
be/src/exec/sink/writer/vtablet_writer_v2.cpp | 11 +-
be/src/exec/sink/writer/vtablet_writer_v2.h | 3 +-
be/src/exprs/function/function_search.cpp | 9 +-
be/src/exprs/runtime_filter_expr.cpp | 9 +-
.../table/iceberg_delete_file_reader_helper.cpp | 3 +-
.../iceberg_position_delete_sys_table_reader.cpp | 608 +
.../iceberg_position_delete_sys_table_reader.h | 110 +
be/src/format_v2/parquet/parquet_file_context.cpp | 10 +-
be/src/format_v2/parquet/parquet_profile.cpp | 3 +
be/src/format_v2/parquet/parquet_profile.h | 2 +
be/src/format_v2/parquet/parquet_scan.cpp | 38 +-
be/src/format_v2/parquet/parquet_scan.h | 5 +
.../parquet/reader/scalar_column_reader.cpp | 1 +
be/src/format_v2/table/hudi_reader.cpp | 12 +-
.../iceberg_position_delete_sys_table_reader.cpp | 596 +
.../iceberg_position_delete_sys_table_reader.h | 86 +
be/src/format_v2/table/iceberg_reader.cpp | 8 +
be/src/format_v2/table/paimon_reader.cpp | 12 +-
be/src/format_v2/table_reader.cpp | 24 +-
be/src/format_v2/table_reader.h | 34 +-
be/src/io/cache/block_file_cache.cpp | 23 +-
be/src/io/cache/block_file_cache_profile.cpp | 28 +
be/src/io/cache/block_file_cache_profile.h | 6 +
be/src/io/cache/block_file_cache_ttl_mgr.cpp | 13 +-
be/src/io/cache/cached_remote_file_reader.cpp | 33 +-
be/src/io/cache/file_cache_common.h | 10 +
.../io/cache/remote_scan_cache_write_limiter.cpp | 78 +
.../src/io/cache/remote_scan_cache_write_limiter.h | 35 +-
be/src/io/file_factory.h | 5 +
be/src/io/io_common.h | 20 +
be/src/load/channel/tablets_channel.cpp | 65 +-
be/src/load/channel/tablets_channel.h | 8 +-
be/src/load/delta_writer/delta_writer.cpp | 6 +-
be/src/load/delta_writer/delta_writer.h | 4 +-
be/src/load/delta_writer/delta_writer_context.h | 7 +
be/src/load/delta_writer/delta_writer_v2.cpp | 6 +-
be/src/load/delta_writer/delta_writer_v2.h | 2 +-
be/src/load/memtable/memtable.cpp | 70 +-
be/src/load/memtable/memtable.h | 20 +-
be/src/load/memtable/memtable_flush_executor.cpp | 35 +-
be/src/load/memtable/memtable_flush_executor.h | 9 -
be/src/load/memtable/memtable_writer.cpp | 44 +-
be/src/load/memtable/memtable_writer.h | 3 +-
be/src/runtime/query_context.cpp | 19 +-
be/src/runtime/query_context.h | 9 +
be/src/runtime/workload_group/workload_group.cpp | 56 +-
be/src/service/point_query_executor.cpp | 38 +-
be/src/service/point_query_executor.h | 4 +
be/src/storage/binlog.h | 26 +-
.../compaction/cumulative_compaction_policy.cpp | 16 +-
.../index/inverted/inverted_index_fs_directory.cpp | 4 +
.../index/inverted/util/term_position_iterator.h | 2 +-
be/src/storage/index/primary_key_index.cpp | 27 +-
be/src/storage/index/primary_key_index.h | 12 +-
be/src/storage/iterator/block_reader.cpp | 22 +-
be/src/storage/iterator/block_reader.h | 4 +-
be/src/storage/iterator/vcollect_iterator.cpp | 10 +-
be/src/storage/iterator/vcollect_iterator.h | 2 +-
be/src/storage/iterator/vgeneric_iterators.cpp | 2 +-
be/src/storage/iterator/vgeneric_iterators.h | 4 +-
be/src/storage/iterators.h | 6 +-
be/src/storage/rowset/beta_rowset.cpp | 20 +-
be/src/storage/rowset/beta_rowset.h | 7 +-
be/src/storage/rowset/beta_rowset_reader.cpp | 20 +-
be/src/storage/rowset/beta_rowset_writer.cpp | 4 +
be/src/storage/rowset/beta_rowset_writer.h | 1 +
be/src/storage/rowset/beta_rowset_writer_v2.cpp | 4 +
be/src/storage/rowset/beta_rowset_writer_v2.h | 1 +
be/src/storage/rowset/group_rowset_writer.cpp | 17 +-
be/src/storage/rowset/group_rowset_writer.h | 7 +-
be/src/storage/rowset/rowset_writer.h | 5 +
be/src/storage/schema.cpp | 5 +-
be/src/storage/schema.h | 20 +-
be/src/storage/segment/column_meta_accessor.cpp | 65 +-
be/src/storage/segment/column_meta_accessor.h | 14 +-
be/src/storage/segment/column_reader_cache.cpp | 23 +-
be/src/storage/segment/column_reader_cache.h | 25 +-
be/src/storage/segment/external_col_meta_util.cpp | 25 +-
be/src/storage/segment/external_col_meta_util.h | 6 +-
.../storage/segment/lazy_init_segment_iterator.cpp | 3 +-
.../storage/segment/row_binlog_segment_writer.cpp | 47 +-
be/src/storage/segment/row_binlog_segment_writer.h | 4 +-
be/src/storage/segment/segment.cpp | 161 +-
be/src/storage/segment/segment.h | 39 +-
be/src/storage/segment/segment_iterator.cpp | 80 +-
be/src/storage/segment/segment_iterator.h | 1 -
be/src/storage/segment/segment_loader.cpp | 12 +-
be/src/storage/segment/segment_loader.h | 6 +-
.../segment/variant/hierarchical_data_iterator.cpp | 12 +-
.../segment/variant/hierarchical_data_iterator.h | 5 +-
.../segment/variant/variant_column_reader.cpp | 107 +-
.../segment/variant/variant_column_reader.h | 31 +-
.../variant/variant_external_meta_reader.cpp | 41 +-
.../segment/variant/variant_external_meta_reader.h | 19 +-
be/src/storage/tablet/base_tablet.cpp | 79 +-
be/src/storage/tablet/base_tablet.h | 6 +-
be/src/storage/tablet/tablet_meta.cpp | 14 +-
be/src/storage/tablet/tablet_schema.cpp | 47 +-
be/src/storage/tablet/tablet_schema.h | 22 +-
be/src/storage/utils.h | 5 +-
be/src/util/threadpool.cpp | 5 +
be/src/util/work_thread_pool.hpp | 3 +
.../cloud_cumulative_compaction_policy_test.cpp | 118 +-
be/test/core/data_type/block_check_type.cpp | 22 +
.../data_type_serde_arrow_validation_test.cpp | 275 +
.../runtime_filter_expr_sampling_test.cpp | 35 +
be/test/exec/scan/file_scanner_v2_test.cpp | 213 +
be/test/exec/scan/olap_scanner_test.cpp | 107 +
be/test/exec/sink/vtablet_writer_v2_test.cpp | 2 +-
.../iceberg_delete_file_reader_helper_test.cpp | 85 +-
...eberg_position_delete_sys_table_reader_test.cpp | 606 +
be/test/format_v2/jni/jni_table_reader_test.cpp | 1 +
be/test/format_v2/parquet/parquet_reader_test.cpp | 79 +-
be/test/format_v2/parquet/parquet_scan_test.cpp | 133 +
be/test/format_v2/table/hudi_reader_test.cpp | 14 +
be/test/format_v2/table/iceberg_reader_test.cpp | 16 +
be/test/format_v2/table/paimon_reader_test.cpp | 15 +
be/test/format_v2/table_reader_test.cpp | 61 +-
.../block_file_cache_profile_reporter_test.cpp | 5 +
be/test/io/cache/block_file_cache_test.cpp | 516 +
be/test/io/cache/block_file_cache_ttl_mgr_test.cpp | 126 +-
.../delta_writer/delta_writer_cluster_key_test.cpp | 10 +-
be/test/load/delta_writer/delta_writer_test.cpp | 14 +-
.../load/memtable/memtable_flush_executor_test.cpp | 43 +-
.../load/memtable/memtable_memory_limiter_test.cpp | 2 +-
be/test/olap/rowset/group_rowset_builder_test.cpp | 2 +-
be/test/olap/rowset/group_rowset_writer_test.cpp | 4 +-
be/test/runtime/snapshot_loader_test.cpp | 2 +-
.../cumulative_compaction_policy_test.cpp | 230 +-
be/test/storage/index/primary_key_index_test.cpp | 111 +
.../iterator/binlog_block_reader_utils_test.cpp | 4 +-
.../predicate/block_column_predicate_test.cpp | 2 +-
.../storage/segment/column_reader_cache_test.cpp | 9 +-
.../segment/external_col_meta_util_test.cpp | 163 +
.../segment/inverted_index_fs_directory_test.cpp | 58 +-
be/test/storage/segment/mock/mock_segment.h | 3 +-
be/test/storage/segment/segment_cache_test.cpp | 4 +-
.../storage/segment/segment_footer_cache_test.cpp | 104 +
.../segment_iterator_no_need_read_data_test.cpp | 38 +
be/test/storage/tablet/tablet_cooldown_test.cpp | 2 +-
.../index_storage_variant_io_context_test.cpp | 113 +
.../variant/variant_column_writer_reader_test.cpp | 14 +-
be/test/testutil/creators.h | 17 +-
docker/runtime/doris-compose/utils.py | 36 +-
fe/fe-authentication/fe-authentication-api/pom.xml | 4 +-
.../fe-authentication-role-mapping/pom.xml | 4 +-
.../main/java/org/apache/doris/catalog/Column.java | 2 +-
.../apache/doris/job/cdc/DataSourceConfigKeys.java | 1 +
.../iceberg/IcebergConnectorMetadata.java | 36 +-
.../connector/iceberg/IcebergPartitionUtils.java | 120 +
.../connector/iceberg/IcebergScanPlanProvider.java | 249 +-
.../doris/connector/iceberg/IcebergScanRange.java | 113 +-
.../IcebergConnectorMetadataSysTableTest.java | 67 +-
.../iceberg/IcebergPartitionUtilsTest.java | 159 +
.../iceberg/IcebergScanPlanProviderTest.java | 167 +
.../apache/doris/alter/SchemaChangeHandler.java | 4 +-
.../java/org/apache/doris/catalog/OlapTable.java | 10 +-
.../doris/catalog/RowBinlogTableWrapper.java | 4 +-
.../apache/doris/cloud/CacheHotspotManager.java | 40 +-
.../org/apache/doris/cloud/CloudWarmUpJob.java | 15 +-
.../cloud/catalog/CloudInstanceStatusChecker.java | 19 +-
.../datasource/iceberg/source/IcebergSplit.java | 17 +
.../doris/datasource/systable/IcebergSysTable.java | 15 +-
.../RangerDorisAccessControllerFactory.java | 7 +-
.../doris/nereids/rules/analysis/BindRelation.java | 2 +-
.../rules/implementation/SplitAggMultiPhase.java | 2 +-
.../SplitAggMultiPhaseWithoutGbyKey.java | 2 +-
.../nereids/rules/rewrite/CheckMultiDistinct.java | 50 +-
.../rules/rewrite/DistinctAggStrategySelector.java | 3 +-
.../rules/rewrite/DistinctAggregateRewriter.java | 67 +-
.../rewrite/NormalizeOlapTableStreamScan.java | 2 +-
.../expressions/functions/agg/GroupConcat.java | 8 +
.../functions/agg/MultiDistinctGroupConcat.java | 8 +
.../trees/plans/commands/ExecuteCommand.java | 2 +-
.../trees/plans/commands/info/CreateIndexOp.java | 6 +
.../trees/plans/commands/info/DropIndexOp.java | 6 +
.../apache/doris/nereids/util/AggregateUtils.java | 20 +-
.../java/org/apache/doris/qe/SessionVariable.java | 13 +-
.../apache/doris/qe/ShortCircuitQueryContext.java | 26 +-
.../org/apache/doris/qe/runtime/LoadProcessor.java | 22 +-
.../qe/runtime/SingleFragmentPipelineTask.java | 19 +-
.../CdcStreamTableValuedFunction.java | 1 +
.../doris/tablefunction/TableBinlogFunction.java | 2 +-
.../org/apache/doris/task/CreateReplicaTask.java | 16 +
.../doris/alter/SchemaChangeHandlerTest.java | 12 +-
.../catalog/OlapTableRowBinlogSchemaTest.java | 12 +-
.../cloud/CacheHotspotManagerTableFilterTest.java | 61 +
.../doris/cloud/cache/CacheHotspotManagerTest.java | 46 +
.../catalog/CloudInstanceStatusCheckerTest.java | 5 +-
.../systable/IcebergSysTableResolverTest.java | 4 +-
.../RangerDorisAccessControllerFactoryTest.java} | 27 +-
.../rewrite/DistinctAggregateRewriterTest.java | 13 +-
.../org/apache/doris/qe/SessionVariablesTest.java | 13 +
.../doris/qe/ShortCircuitQueryContextTest.java | 58 +
.../qe/runtime/SingleFragmentPipelineTaskTest.java | 96 +
.../cdcclient/service/PipelineCoordinator.java | 7 +-
.../deserialize/DebeziumJsonDeserializer.java | 5 +
.../deserialize/MySqlDebeziumJsonDeserializer.java | 193 +-
.../PostgresDebeziumJsonDeserializer.java | 19 +-
.../mysql/CustomAlterTableParserListener.java | 162 +
.../parse/mysql/CustomMySqlAntlrDdlParser.java | 282 +
.../mysql/CustomMySqlAntlrDdlParserListener.java | 118 +
.../source/parse/mysql/MySqlSchemaChange.java | 90 +
.../source/reader/mysql/MySqlSourceReader.java | 14 +-
.../reader/postgres/PostgresSourceReader.java | 5 +-
.../doris/cdcclient/utils/SchemaChangeHelper.java | 202 +-
.../doris/cdcclient/itcase/MockDorisServer.java | 68 +-
.../itcase/MySqlWriteSchemaChangeITCase.java | 154 +
.../MySqlSchemaChangeDeserializeTest.java | 188 +
.../PostgresSchemaChangeDeserializeTest.java | 14 +
.../parse/mysql/CustomMySqlAntlrDdlParserTest.java | 152 +
.../source/reader/mysql/MySqlSourceReaderTest.java | 34 +-
.../reader/postgres/PostgresSourceReaderTest.java | 43 +
.../cdcclient/utils/SchemaChangeHelperTest.java | 75 +-
gensrc/proto/internal_service.proto | 3 +
gensrc/proto/olap_file.proto | 26 +-
gensrc/thrift/AgentService.thrift | 3 +
gensrc/thrift/PaloInternalService.thrift | 1 +
gensrc/thrift/PlanNodes.thrift | 4 +
plan-doc/HANDOFF.md | 62 +-
.../P6.6-position-deletes-connector-port-design.md | 249 +
...sition-deletes-connector-port-research-notes.md | 118 +
.../decimalv2/test_decimalv2_common.out | 22479 -------------------
.../test_iceberg_position_deletes_sys_table.out | 4 +
.../test_zonemap_always_true_count_on_index.out | 7 +
.../cdc/test_streaming_mysql_job_sc.out | 18 +
.../cdc/test_streaming_mysql_job_sc_advanced.out | 39 +
...test_streaming_mysql_job_sc_during_snapshot.out | 9 +
.../cdc/test_streaming_mysql_job_sc_latest.out | 9 +
.../cdc/test_streaming_mysql_job_sc_restart_fe.out | 11 +
...treaming_job_cdc_stream_mysql_schema_change.out | 4 +
...aming_job_cdc_stream_postgres_schema_change.out | 4 +
.../agg_strategy/distinct_agg_rewriter.out | 9 +
.../distinct_agg_strategy_selector.out | 15 +
.../join/test_hash_join_probe_side_zero_copy.out | 16 +
.../data/row_binlog_p0/test_row_binlog_basic.out | 6 +
.../data/search/test_search_score_cache.out | 42 +
.../regression/util/WarmupMetricsUtils.groovy | 310 +-
.../nonConcurrent/conf/regression-conf.groovy | 1 -
...p_cluster_event_conflict_and_diagnostics.groovy | 111 +
...test_warm_up_cluster_periodic_add_new_be.groovy | 184 +-
..._up_cluster_periodic_scheduler_semantics.groovy | 137 +
...arm_up_mixed_cluster_change_and_failover.groovy | 117 +
.../test_warm_up_normal_queue_semantics.groovy | 146 +
...arm_up_event_on_tables_overlap_semantics.groovy | 125 +
...e_cache_query_limit_segment_meta_profile.groovy | 475 +
...emote_scan_no_write_file_cache_threshold.groovy | 648 +
...test_cloud_alter_disable_auto_compaction.groovy | 8 +-
...t_vcg_warmup_allows_manual_periodic_jobs.groovy | 103 +
...est_vcg_warmup_failover_cancels_old_jobs.groovy | 125 +
...test_vcg_warmup_precedence_and_reconcile.groovy | 153 +
.../test_vcg_warmup_restart_master_fe.groovy | 140 +
.../test_vcg_warmup_restart_target_be.groovy | 130 +
...vcg_warmup_shared_compute_group_conflict.groovy | 99 +
...g_warmup_switch_and_drop_cancels_history.groovy | 130 +
...g_warmup_waits_for_existing_cluster_jobs.groovy | 107 +
..._warmup_with_manual_once_queue_semantics.groovy | 111 +
...mup_with_manual_periodic_queue_semantics.groovy | 110 +
.../decimalv2/test_decimalv2_common.groovy | 406 -
.../test_iceberg_position_deletes_sys_table.groovy | 659 +
.../iceberg/test_iceberg_sys_table.groovy | 30 +-
.../test_zonemap_always_true_count_on_index.groovy | 83 +
.../cdc/test_streaming_mysql_job_sc.groovy | 166 +
.../test_streaming_mysql_job_sc_advanced.groovy | 198 +
...t_streaming_mysql_job_sc_during_snapshot.groovy | 147 +
.../cdc/test_streaming_mysql_job_sc_latest.groovy | 125 +
.../test_streaming_mysql_job_sc_restart_fe.groovy | 153 +
...aming_job_cdc_stream_mysql_schema_change.groovy | 116 +
...ng_job_cdc_stream_postgres_schema_change.groovy | 119 +
.../test_insert_fail_fast_after_be_restart.groovy | 128 +
.../agg_strategy/distinct_agg_rewriter.groovy | 23 +-
.../distinct_agg_strategy_selector.groovy | 4 +
.../query_p0/aggregate/agg_group_concat.groovy | 9 +
.../query_p0/group_concat/test_group_concat.groovy | 2 +-
.../test_hash_join_probe_side_zero_copy.groovy | 111 +
.../test_binlog_changes_syntax.groovy | 38 +-
.../row_binlog_p0/test_row_binlog_basic.groovy | 48 +-
.../test_row_binlog_multi_segment.groovy | 8 +-
.../test_row_binlog_publish_conflict.groovy | 8 +-
.../test_row_binlog_schema_change.groovy | 6 +-
.../suites/search/test_search_cache.groovy | 30 +
.../suites/search/test_search_score_cache.groovy | 254 +
323 files changed, 17010 insertions(+), 24130 deletions(-)
create mode 100644 be/benchmark/benchmark_arrow_validation.hpp
create mode 100644 be/src/core/data_type_serde/arrow_validation.h
create mode 100644 be/src/exec/scan/file_scan_io_context.h
create mode 100644
be/src/format/table/iceberg_position_delete_sys_table_reader.cpp
create mode 100644
be/src/format/table/iceberg_position_delete_sys_table_reader.h
create mode 100644
be/src/format_v2/table/iceberg_position_delete_sys_table_reader.cpp
create mode 100644
be/src/format_v2/table/iceberg_position_delete_sys_table_reader.h
create mode 100644 be/src/io/cache/remote_scan_cache_write_limiter.cpp
copy
fe/fe-core/src/test/java/org/apache/doris/datasource/systable/IcebergSysTableResolverTest.java
=> be/src/io/cache/remote_scan_cache_write_limiter.h (52%)
create mode 100644
be/test/core/data_type_serde/data_type_serde_arrow_validation_test.cpp
create mode 100644 be/test/exec/scan/olap_scanner_test.cpp
create mode 100644
be/test/format/table/iceberg/iceberg_position_delete_sys_table_reader_test.cpp
create mode 100644
be/test/storage/variant/index_storage_variant_io_context_test.cpp
copy
fe/fe-core/src/{main/java/org/apache/doris/mysql/privilege/RangerDorisAccessControllerFactory.java
=>
test/java/org/apache/doris/mysql/privilege/RangerDorisAccessControllerFactoryTest.java}
(51%)
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/qe/ShortCircuitQueryContextTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/qe/runtime/SingleFragmentPipelineTaskTest.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/parse/mysql/CustomAlterTableParserListener.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/parse/mysql/CustomMySqlAntlrDdlParser.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/parse/mysql/CustomMySqlAntlrDdlParserListener.java
create mode 100644
fs_brokers/cdc_client/src/main/java/org/apache/doris/cdcclient/source/parse/mysql/MySqlSchemaChange.java
create mode 100644
fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/itcase/MySqlWriteSchemaChangeITCase.java
create mode 100644
fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/deserialize/MySqlSchemaChangeDeserializeTest.java
create mode 100644
fs_brokers/cdc_client/src/test/java/org/apache/doris/cdcclient/source/parse/mysql/CustomMySqlAntlrDdlParserTest.java
create mode 100644
plan-doc/tasks/designs/P6.6-position-deletes-connector-port-design.md
create mode 100644
plan-doc/tasks/designs/P6.6-position-deletes-connector-port-research-notes.md
delete mode 100644
regression-test/data/datatype_p0/decimalv2/test_decimalv2_common.out
create mode 100644
regression-test/data/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.out
create mode 100644
regression-test/data/fault_injection_p0/test_zonemap_always_true_count_on_index.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_sc.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_sc_advanced.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_sc_during_snapshot.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_sc_latest.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/test_streaming_mysql_job_sc_restart_fe.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_mysql_schema_change.out
create mode 100644
regression-test/data/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_schema_change.out
create mode 100644
regression-test/data/query_p0/join/test_hash_join_probe_side_zero_copy.out
create mode 100644 regression-test/data/search/test_search_score_cache.out
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_event_conflict_and_diagnostics.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_cluster_periodic_scheduler_semantics.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_mixed_cluster_change_and_failover.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/cluster/test_warm_up_normal_queue_semantics.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/multi_cluster/warm_up/on_tables/test_warm_up_event_on_tables_overlap_semantics.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/remote_scan_no_write_file_cache/test_file_cache_query_limit_segment_meta_profile.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/remote_scan_no_write_file_cache/test_remote_scan_no_write_file_cache_threshold.groovy
create mode 100644
regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_warmup_allows_manual_periodic_jobs.groovy
create mode 100644
regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_warmup_failover_cancels_old_jobs.groovy
create mode 100644
regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_warmup_precedence_and_reconcile.groovy
create mode 100644
regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_warmup_restart_master_fe.groovy
create mode 100644
regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_warmup_restart_target_be.groovy
create mode 100644
regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_warmup_shared_compute_group_conflict.groovy
create mode 100644
regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_warmup_switch_and_drop_cancels_history.groovy
create mode 100644
regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_warmup_waits_for_existing_cluster_jobs.groovy
create mode 100644
regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_warmup_with_manual_once_queue_semantics.groovy
create mode 100644
regression-test/suites/cloud_p0/multi_cluster/virtual_compute_group/test_vcg_warmup_with_manual_periodic_queue_semantics.groovy
delete mode 100644
regression-test/suites/datatype_p0/decimalv2/test_decimalv2_common.groovy
create mode 100644
regression-test/suites/external_table_p0/iceberg/test_iceberg_position_deletes_sys_table.groovy
create mode 100644
regression-test/suites/fault_injection_p0/test_zonemap_always_true_count_on_index.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_sc.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_sc_advanced.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_sc_during_snapshot.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_sc_latest.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/test_streaming_mysql_job_sc_restart_fe.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_mysql_schema_change.groovy
create mode 100644
regression-test/suites/job_p0/streaming_job/cdc/tvf/test_streaming_job_cdc_stream_postgres_schema_change.groovy
create mode 100644
regression-test/suites/load_p0/insert/test_insert_fail_fast_after_be_restart.groovy
create mode 100644
regression-test/suites/query_p0/join/test_hash_join_probe_side_zero_copy.groovy
create mode 100644 regression-test/suites/search/test_search_score_cache.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
