This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch auto-pick-66460-branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
omit 75f6a04c63a [fix](regression) Fetch streaming job metrics from master
FE (#66460)
add 06ffaef2520 branch-4.1: [feature](file-cache) Disable cache writes
after remote scan threshold (#66479)
add 3ba435c26e0 branch-4.1: [fix](be) Clean up spill directories during
query teardown (#66458)
add a15b79e4940 branch-4.1: [improvement](be) Add instance ID to hash join
profiles #66097 (#66453)
add e48fc7e97b9 [fix](test) Unset Paimon Variant shredding property
(#66536)
add f0e00809c81 branch-4.1:[chore](thirdparty) Upgrade Apache Arrow to
24.0.0(#66221) (#66480)
add ca02e7b3839 [fix](regression) Stabilize external table guardrail cases
(#66509)
add c3cfb3805d4 [opt](paimon) report error when mapping variant type
disable variant_v2 (#66466)
add d39e68c81ed [fix](be) Preserve floating-point equality in Parquet
pruning (#66470)
add 6c8ccecbd1e branch-4.1: [improvement](repeat) Optimize shuffle key
selection for Repeat decomposition #66532 (#66568)
add 762147325b7 branch-4.1: [fix](topn) Handle empty rowid fetch RPC
failures #66443 (#66522)
add 895c29374e0 branch-4.1: [Feature](multimodal_embed) Support
multimodal(file) embed (#66461)
add 6a4700877d6 [opt](paimon) support variant access path in paimon jni
reader (#66547)
add 39c66a37715 [fix](regression) Fetch streaming job metrics from master
FE (#66460)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (75f6a04c63a)
\
N -- N -- N refs/heads/auto-pick-66460-branch-4.1 (39c66a37715)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.github/workflows/be-ut-mac.yml | 50 +-
.github/workflows/build-thirdparty.yml | 38 +-
be/CMakeLists.txt | 2 +-
be/cmake/thirdparty.cmake | 1 +
be/src/common/config.cpp | 2 +
be/src/common/config.h | 1 +
be/src/core/field.h | 13 +
be/src/exec/operator/hashjoin_build_sink.cpp | 1 +
be/src/exec/operator/hashjoin_probe_operator.cpp | 2 +
be/src/exec/operator/materialization_opertor.cpp | 57 +-
be/src/exec/operator/materialization_opertor.h | 1 +
.../scan/file_scan_io_context.h} | 39 +-
be/src/exec/scan/file_scanner.h | 4 +-
be/src/exec/scan/file_scanner_v2.cpp | 4 +-
be/src/exec/scan/olap_scanner.cpp | 27 +-
be/src/exec/scan/olap_scanner.h | 9 +
be/src/exec/scan/parallel_scanner_builder.cpp | 21 +-
be/src/exec/spill/spill_file.cpp | 4 +-
be/src/exec/spill/spill_file_manager.cpp | 85 +-
be/src/exec/spill/spill_file_manager.h | 25 +-
be/src/exec/spill/spill_file_writer.cpp | 3 +
be/src/exprs/expr_zonemap_filter.cpp | 37 +-
be/src/exprs/expr_zonemap_filter.h | 4 +-
be/src/exprs/function/ai/ai_adapter.h | 433 ++++++-
be/src/exprs/function/ai/ai_classify.h | 4 +-
be/src/exprs/function/ai/ai_extract.h | 4 +-
be/src/exprs/function/ai/ai_filter.h | 4 +-
be/src/exprs/function/ai/ai_fix_grammar.h | 2 +-
be/src/exprs/function/ai/ai_functions.cpp | 164 +--
be/src/exprs/function/ai/ai_functions.h | 215 ++--
be/src/exprs/function/ai/ai_generate.h | 4 +-
be/src/exprs/function/ai/ai_mask.h | 4 +-
be/src/exprs/function/ai/ai_sentiment.h | 2 +-
be/src/exprs/function/ai/ai_similarity.h | 4 +-
be/src/exprs/function/ai/ai_summarize.h | 2 +-
be/src/exprs/function/ai/ai_translate.h | 4 +-
be/src/exprs/function/ai/embed.h | 371 +++++-
be/src/exprs/function/functions_comparison.h | 17 +-
be/src/exprs/hybrid_set.h | 18 +-
be/src/exprs/vdirect_in_predicate.h | 5 +-
be/src/exprs/vin_predicate.cpp | 8 +-
be/src/exprs/vin_predicate.h | 1 +
be/src/format/jni/jni_data_bridge.cpp | 2 +-
be/src/format/jni/jni_data_bridge.h | 4 +-
be/src/format/table/iceberg/arrow_schema_util.cpp | 2 +-
.../table/iceberg_delete_file_reader_helper.cpp | 8 +-
be/src/format/table/paimon_jni_reader.cpp | 4 +-
be/src/format_v2/jni/jni_table_reader.cpp | 4 +-
be/src/format_v2/jni/paimon_jni_reader.cpp | 13 +
be/src/format_v2/parquet/native_schema_desc.cpp | 17 +-
be/src/format_v2/parquet/native_schema_desc.h | 2 +-
be/src/format_v2/parquet/parquet_statistics.cpp | 3 +
be/src/format_v2/table/iceberg_reader.cpp | 92 --
be/src/format_v2/table/iceberg_reader.h | 4 -
be/src/format_v2/table_reader.cpp | 60 +
be/src/format_v2/table_reader.h | 4 +-
be/src/io/cache/block_file_cache.cpp | 23 +-
be/src/io/cache/block_file_cache_profile.cpp | 28 +
be/src/io/cache/block_file_cache_profile.h | 6 +
be/src/io/cache/cached_remote_file_reader.cpp | 33 +-
be/src/io/cache/file_cache_common.h | 10 +
.../io/cache/remote_scan_cache_write_limiter.cpp | 78 ++
.../remote_scan_cache_write_limiter.h} | 26 +-
be/src/io/io_common.h | 20 +
be/src/runtime/query_context.cpp | 23 +
be/src/runtime/query_context.h | 18 +
be/src/service/point_query_executor.cpp | 38 +-
be/src/service/point_query_executor.h | 4 +
.../index/inverted/inverted_index_fs_directory.cpp | 4 +
.../index/inverted/util/term_position_iterator.h | 2 +-
be/src/storage/index/primary_key_index.cpp | 27 +-
be/src/storage/index/primary_key_index.h | 12 +-
.../index/zone_map/zonemap_eval_context.cpp | 5 +
.../storage/index/zone_map/zonemap_eval_context.h | 3 +
be/src/storage/rowset/beta_rowset.cpp | 20 +-
be/src/storage/rowset/beta_rowset.h | 7 +-
be/src/storage/rowset/beta_rowset_reader.cpp | 6 +
be/src/storage/segment/column_meta_accessor.cpp | 65 +-
be/src/storage/segment/column_meta_accessor.h | 14 +-
be/src/storage/segment/column_reader_cache.cpp | 25 +-
be/src/storage/segment/column_reader_cache.h | 25 +-
be/src/storage/segment/external_col_meta_util.cpp | 25 +-
be/src/storage/segment/external_col_meta_util.h | 6 +-
.../storage/segment/lazy_init_segment_iterator.cpp | 3 +-
be/src/storage/segment/segment.cpp | 147 ++-
be/src/storage/segment/segment.h | 37 +-
be/src/storage/segment/segment_iterator.cpp | 5 +-
be/src/storage/segment/segment_loader.cpp | 12 +-
be/src/storage/segment/segment_loader.h | 6 +-
.../segment/variant/hierarchical_data_iterator.cpp | 12 +-
.../segment/variant/hierarchical_data_iterator.h | 5 +-
.../segment/variant/variant_column_reader.cpp | 107 +-
.../segment/variant/variant_column_reader.h | 31 +-
.../variant/variant_external_meta_reader.cpp | 41 +-
.../segment/variant/variant_external_meta_reader.h | 19 +-
be/src/storage/tablet/base_tablet.cpp | 10 +-
be/src/storage/tablet/base_tablet.h | 3 +-
be/test/ai/ai_adapter_test.cpp | 423 ++++++-
be/test/ai/ai_function_test.cpp | 1236 ++++++++++++++++++--
be/test/ai/embed_test.cpp | 836 ++++++++++++-
.../data_type_serde_decimal_test.cpp | 4 +-
be/test/core/field_test.cpp | 11 +
.../operator/materialization_shared_state_test.cpp | 41 +-
be/test/exec/scan/file_scanner_v2_test.cpp | 83 ++
be/test/exec/scan/olap_scanner_test.cpp | 107 ++
be/test/exprs/expr_zonemap_filter_test.cpp | 223 +++-
be/test/exprs/hybrid_set_test.cpp | 29 +
be/test/format/parquet/parquet_expr_test.cpp | 2 +-
.../iceberg_delete_file_reader_helper_test.cpp | 43 +-
be/test/format_v2/jni/jni_table_reader_test.cpp | 6 +-
be/test/format_v2/jni/paimon_jni_reader_test.cpp | 29 +-
be/test/format_v2/parquet/parquet_reader_test.cpp | 58 +-
be/test/format_v2/parquet/parquet_scan_test.cpp | 241 ++++
be/test/format_v2/parquet/parquet_schema_test.cpp | 36 +-
.../format_v2/parquet/parquet_statistics_test.cpp | 330 ++++++
be/test/format_v2/table/iceberg_reader_test.cpp | 39 -
.../format_v2/table/remote_doris_reader_test.cpp | 2 +-
be/test/format_v2/table_reader_test.cpp | 40 +
.../block_file_cache_profile_reporter_test.cpp | 5 +
be/test/io/cache/block_file_cache_test.cpp | 516 ++++++++
be/test/storage/index/primary_key_index_test.cpp | 111 ++
.../storage/segment/column_reader_cache_test.cpp | 9 +-
.../segment/external_col_meta_util_test.cpp | 163 +++
.../segment/inverted_index_fs_directory_test.cpp | 58 +-
be/test/storage/segment/mock/mock_segment.h | 3 +-
.../storage/segment/segment_footer_cache_test.cpp | 104 ++
.../index_storage_variant_io_context_test.cpp | 113 ++
.../variant/variant_column_writer_reader_test.cpp | 14 +-
be/test/vec/spill/spill_file_test.cpp | 406 ++++++-
build.sh | 75 +-
docker/compilation/Dockerfile.gcc10 | 3 +-
docker/compilation/Dockerfile.gcc7 | 119 --
docker/compilation/arm/Dockerfile | 12 +
.../create_preinstalled_scripts/paimon/run13.sql | 38 +-
.../org/apache/doris/paimon/PaimonColumnValue.java | 28 +-
.../org/apache/doris/paimon/PaimonJniScanner.java | 69 +-
.../doris/paimon/PaimonVariantProjection.java | 228 ++++
.../apache/doris/paimon/PaimonJniScannerTest.java | 14 +
.../doris/paimon/PaimonVariantProjectionTest.java | 147 +++
.../java/org/apache/doris/catalog/Resource.java | 8 +
.../apache/doris/datasource/paimon/PaimonUtil.java | 18 +
.../datasource/paimon/source/PaimonScanNode.java | 12 +
.../property/constants/AIProperties.java | 2 +-
.../org/apache/doris/nereids/StatementContext.java | 17 +-
.../rewrite/DecomposeRepeatWithPreAggregation.java | 13 +-
.../nereids/rules/rewrite/SlotTypeReplacer.java | 8 +-
.../trees/expressions/functions/agg/AIAgg.java | 1 +
.../trees/expressions/functions/ai/AIFunction.java | 1 +
.../trees/expressions/functions/ai/Embed.java | 48 +-
.../trees/plans/commands/ExecuteCommand.java | 9 +-
.../trees/plans/logical/LogicalFileScan.java | 15 +-
.../plans/logical/SupportPruneNestedColumn.java | 7 +
.../main/java/org/apache/doris/qe/Coordinator.java | 26 +-
.../java/org/apache/doris/qe/SessionVariable.java | 21 +
.../apache/doris/qe/ShortCircuitQueryContext.java | 24 +
.../doris/qe/runtime/ThriftPlansBuilder.java | 30 +-
.../doris/statistics/util/StatisticsUtil.java | 7 +-
.../paimon/source/PaimonScanNodeTest.java | 32 +
.../DecomposeRepeatWithPreAggregationTest.java | 2 +-
.../trees/plans/logical/LogicalFileScanTest.java | 29 +
.../org/apache/doris/qe/SessionVariablesTest.java | 14 +
.../doris/qe/ShortCircuitQueryContextTest.java | 58 +
gensrc/thrift/PaloInternalService.thrift | 4 +
.../iceberg/write/test_iceberg_write_stats2.out | 2 +-
.../paimon/test_paimon_catalog_variant.out | 31 +
.../paimon/test_paimon_jni_reader_guardrails.out | 6 -
.../suites/ai_p0/test_ai_functions.groovy | 5 +-
...e_cache_query_limit_segment_meta_profile.groovy | 475 ++++++++
...emote_scan_no_write_file_cache_threshold.groovy | 648 ++++++++++
.../iceberg/iceberg_schema_change_ddl.groovy | 2 +-
.../iceberg/test_iceberg_variant_read.groovy | 14 +-
.../paimon/test_paimon_catalog_variant.groovy | 123 ++
.../test_paimon_jni_reader_guardrails.groovy | 7 +-
.../test_remote_doris_variant_select.groovy | 3 +-
.../spill_p0/test_spill_directory_cleanup.groovy | 113 ++
run-be-ut.sh | 25 +
thirdparty/arrow-paimon-vars.sh | 408 +++++++
thirdparty/build-thirdparty.sh | 16 +-
thirdparty/download-thirdparty.sh | 64 +-
thirdparty/paimon-cpp-cache.cmake | 3 +-
...rrow-24.0.0-force-write-int96-timestamps.patch} | 42 +-
...0.0-lzo.patch => apache-arrow-24.0.0-lzo.patch} | 72 +-
...imon.patch => apache-arrow-24.0.0-paimon.patch} | 116 +-
.../paimon-cpp-arrow-24-compatibility.patch | 64 +
.../patches/paimon-cpp-arrow-24-compute.patch | 44 +
thirdparty/test/arrow-paimon-lifecycle-test.sh | 492 ++++++++
thirdparty/vars.sh | 31 +-
187 files changed, 10847 insertions(+), 1325 deletions(-)
copy be/src/{storage/compaction/cold_data_compaction.h =>
exec/scan/file_scan_io_context.h} (54%)
create mode 100644 be/src/io/cache/remote_scan_cache_write_limiter.cpp
copy be/src/io/{fs/connectivity/storage_connectivity_tester.h =>
cache/remote_scan_cache_write_limiter.h} (59%)
create mode 100644 be/test/exec/scan/olap_scanner_test.cpp
create mode 100644
be/test/storage/variant/index_storage_variant_io_context_test.cpp
delete mode 100644 docker/compilation/Dockerfile.gcc7
create mode 100644
fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonVariantProjection.java
create mode 100644
fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonVariantProjectionTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/qe/ShortCircuitQueryContextTest.java
create mode 100644
regression-test/suites/cloud_p0/cache/remote_scan_no_write_file_cache/test_file_cache_query_limit_segment_meta_profile.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/remote_scan_no_write_file_cache/test_remote_scan_no_write_file_cache_threshold.groovy
create mode 100644
regression-test/suites/spill_p0/test_spill_directory_cleanup.groovy
create mode 100644 thirdparty/arrow-paimon-vars.sh
rename
thirdparty/patches/{apache-arrow-17.0.0-force-write-int96-timestamps.patch =>
apache-arrow-24.0.0-force-write-int96-timestamps.patch} (74%)
rename thirdparty/patches/{apache-arrow-17.0.0-lzo.patch =>
apache-arrow-24.0.0-lzo.patch} (55%)
rename thirdparty/patches/{apache-arrow-17.0.0-paimon.patch =>
apache-arrow-24.0.0-paimon.patch} (68%)
create mode 100644 thirdparty/patches/paimon-cpp-arrow-24-compatibility.patch
create mode 100644 thirdparty/patches/paimon-cpp-arrow-24-compute.patch
create mode 100755 thirdparty/test/arrow-paimon-lifecycle-test.sh
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
