This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a change to branch auto-pick-58038-branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from c95c747eb3a Merge branch 'branch-4.0' into auto-pick-58038-branch-4.0
add f05a582ab24 branch-4.0: [feature](cloud) Read peer be cache when
balance in same cluster #56384 (#58089)
add 356c25443c2 branch-4.0: [chore](memtable) use doris::Exception replace
std::runtime_error #58098 (#58102)
add 079996228f7 branch-4.0: [feature](jsonb) json type support group by
and distinct #57679 (#58101)
add 98896679c79 branch-4.0: [fix] remove useless const_cast and explain
const_cast(1) #55943 (#58117)
add 4105d6cc574 branch-4.0: [performance](agg) support count push agg in
no null column #58103 (#58133)
add efa23396645 branch-4.0: [enhancement](wg) print error code when delete
cgroup path failed #58126 (#58139)
add b19ca07bad1 branch-4.0: [fix](jdbc) Skip the default creation check
logic #57983 (#58121)
add 6dce942d889 branch-4.0: [Enhancement](ai) relax the matching
restriction of some AI-Functions #58077 (#58113)
add ff742c44eb1 branch-4.0: [feat](web)Adapted for Spring Boot 3 and Jetty
12 #57710 (#57782)
add 9812408dec8 branch-4.0: [improvement](error msg) prioritize retaining
the error URL and first_error_msg when truncation occurs #56759 (#58145)
add 507f29b2680 branch-4.0: [fix](cloud) avoid broadcast remote read in
topn query #58044 (#58156)
add 71f3e91b20a branch-4.0: [fix](metric) fix backend metrics missing in
fe #57935 (#58160)
add 993888e3fc9 branch-4.0: [fix](cloud) Fix cloud drop tablet tasks pile
up #58131 (#58158)
add 5e90fa2ace5 add 4.0.2-rc01 (#58167)
add 07b413228ba branch-4.0: [test](bloom filter) enhance bloom filter
regression case for recover fe configs #58148 (#58168)
add 1d63c19e3ee branch-4.0:[feature](mtmv)MTMV refresh support multi pct
tables (#58140)
add 31353bad17a branch-4.0: [fix](case) fix group commit case #58142
(#58173)
add 6b1f695f107 branch-4.0: [test](ip) fix ip_cidr_search case #58151
(#58169)
add 0cf45d18e51 branch-4.0: [opt](ann index) Introduce IDSelectorRoaring
for a faster roaring_to_faiss_selector #58095 (#58177)
add d51a15621e7 branch-4.0: [fix](file cache) Fix inverted index cache not
cleaned on compaction failure #58141 (#58165)
add 436a73b231b branch-4.0: [fix](load) Fix undefined behavior in the JSON
reader caused by multiple calls to the get_string method #58107 (#58185)
add 03da50fa6de branch-4.0: [fix](iceberg) only enable dynamic partition
pruning for identity partitions in Iceberg #58033 (#58182)
add b0b7b2a8ab8 branch-4.0: [fix](nereids)where clause should only
contains key column when create mv on unique table #57915 (#58179)
add 4e2dd1c498d branch-4.0: [fix](hive) refresh hive cache when
inconsistency #58074 (#58187)
add 40379b7a8eb branch-4.0: [fix](datatype)Fix unaligned memory access in
read_column_from_arrow #58002 (#58202)
add 4bcc1e91456 Merge branch 'branch-4.0' into auto-pick-58038-branch-4.0
No new revisions were added by this update.
Summary of changes:
be/src/agent/cgroup_cpu_ctl.cpp | 7 +-
be/src/agent/task_worker_pool.cpp | 15 +-
be/src/agent/task_worker_pool.h | 2 +-
be/src/cloud/cloud_backend_service.cpp | 24 +-
be/src/cloud/cloud_internal_service.cpp | 122 ++
be/src/cloud/cloud_internal_service.h | 5 +
be/src/cloud/cloud_meta_mgr.cpp | 13 +-
be/src/cloud/cloud_meta_mgr.h | 6 +-
be/src/cloud/cloud_storage_engine.cpp | 4 +-
be/src/cloud/cloud_storage_engine.h | 2 +-
be/src/cloud/cloud_tablet_mgr.cpp | 11 +-
be/src/cloud/cloud_warm_up_manager.cpp | 84 ++
be/src/cloud/cloud_warm_up_manager.h | 28 +
be/src/cloud/config.cpp | 6 +
be/src/cloud/config.h | 4 +
be/src/exec/decompressor.cpp | 2 +-
be/src/exec/table_connector.cpp | 2 +-
be/src/exprs/hybrid_set.h | 17 +-
be/src/geo/geo_types.cpp | 6 +-
be/src/geo/geo_types.h | 2 +-
be/src/http/action/file_cache_action.cpp | 5 +
be/src/io/cache/block_file_cache_downloader.cpp | 43 +-
be/src/io/cache/block_file_cache_factory.cpp | 35 +
be/src/io/cache/block_file_cache_factory.h | 6 +
be/src/io/cache/block_file_cache_profile.cpp | 25 +-
be/src/io/cache/block_file_cache_profile.h | 7 +
be/src/io/cache/cached_remote_file_reader.cpp | 184 ++-
be/src/io/cache/cached_remote_file_reader.h | 13 +-
be/src/io/cache/file_cache_common.h | 2 +
be/src/io/cache/peer_file_cache_reader.cpp | 167 +++
be/src/io/cache/peer_file_cache_reader.h | 82 ++
be/src/io/fs/file_reader.h | 2 +
be/src/io/fs/s3_file_reader.cpp | 17 +-
be/src/io/io_common.h | 6 +
be/src/olap/compaction.cpp | 7 +
be/src/olap/comparison_predicate.h | 18 +-
be/src/olap/in_list_predicate.h | 15 +-
be/src/olap/memtable.cpp | 5 +-
be/src/olap/rowset/beta_rowset.cpp | 1 +
.../segment_v2/ann_index/faiss_ann_index.cpp | 28 +-
.../olap/rowset/segment_v2/index_file_writer.cpp | 14 +
be/src/olap/rowset/segment_v2/index_file_writer.h | 1 +
be/src/olap/storage_engine.cpp | 2 +-
be/src/olap/storage_engine.h | 4 +-
be/src/olap/storage_policy.cpp | 86 ++
be/src/olap/storage_policy.h | 3 +
be/src/runtime/exec_env.cpp | 4 +-
be/src/runtime/exec_env.h | 2 +-
be/src/runtime/memory/memory_profile.cpp | 13 +-
be/src/runtime/process_profile.h | 2 +-
.../runtime/workload_management/resource_context.h | 2 -
be/src/util/doris_metrics.cpp | 2 +
be/src/util/doris_metrics.h | 1 +
be/src/util/runtime_profile.cpp | 6 +-
be/src/util/runtime_profile.h | 4 +-
be/src/vec/common/hash_table/hash_key_type.h | 3 +-
.../vec/data_types/serde/data_type_array_serde.cpp | 16 +-
.../serde/data_type_datetimev2_serde.cpp | 7 +-
.../data_types/serde/data_type_datev2_serde.cpp | 8 +-
.../data_types/serde/data_type_decimal_serde.cpp | 6 +-
.../data_types/serde/data_type_number_serde.cpp | 11 +-
.../data_types/serde/data_type_string_serde.cpp | 15 +-
be/src/vec/exec/format/json/new_json_reader.cpp | 34 +-
be/src/vec/exec/format/json/new_json_reader.h | 20 +
be/src/vec/functions/ai/ai_functions.h | 38 +-
be/test/ai/ai_function_test.cpp | 154 +++
.../segment_v2/inverted_index_file_writer_test.cpp | 49 +
be/test/olap/storage_resource_test.cpp | 109 ++
.../olap/vector_search/faiss_vector_index_test.cpp | 50 +
.../serde/data_type_serde_datetime_v2_test.cpp | 66 +-
.../serde/data_type_serde_decimal_test.cpp | 48 +-
.../serde/data_type_serde_number_test.cpp | 53 +-
.../serde/data_type_serde_string_test.cpp | 129 +-
bin/start_be.sh | 10 +-
build.sh | 51 +-
cloud/script/start.sh | 11 +-
dist/LICENSE-dist.txt | 4 -
.../create_preinstalled_scripts/iceberg/run19.sql | 87 ++
fe/be-java-extensions/avro-scanner/pom.xml | 8 +-
.../{avro-scanner => hadoop-deps}/pom.xml | 93 +-
fe/be-java-extensions/hadoop-hudi-scanner/pom.xml | 39 +-
.../iceberg-metadata-scanner/pom.xml | 11 +-
fe/be-java-extensions/java-udf/pom.xml | 3 +-
fe/be-java-extensions/jdbc-scanner/pom.xml | 3 +-
fe/be-java-extensions/max-compute-scanner/pom.xml | 8 +-
fe/be-java-extensions/paimon-scanner/pom.xml | 10 +-
fe/be-java-extensions/pom.xml | 1 +
fe/be-java-extensions/preload-extensions/pom.xml | 20 +-
.../trino-connector-scanner/pom.xml | 3 +-
fe/check/checkstyle/checkstyle.xml | 6 +
fe/fe-common/pom.xml | 27 -
.../main/java/org/apache/doris/common/Config.java | 7 +-
fe/fe-core/pom.xml | 106 +-
.../apache/doris/alter/SchemaChangeHandler.java | 2 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 2 +-
.../java/org/apache/doris/analysis/BrokerDesc.java | 2 +-
.../org/apache/doris/analysis/CreateTableStmt.java | 2 +-
.../org/apache/doris/analysis/GroupByClause.java | 2 +-
.../java/org/apache/doris/analysis/SlotRef.java | 2 +-
.../java/org/apache/doris/analysis/TableRef.java | 2 +-
.../org/apache/doris/backup/BackupHandler.java | 2 +-
.../java/org/apache/doris/backup/RestoreJob.java | 2 +-
.../doris/catalog/AccessPrivilegeWithCols.java | 2 +-
.../main/java/org/apache/doris/catalog/Column.java | 2 +-
.../main/java/org/apache/doris/catalog/Env.java | 10 +-
.../org/apache/doris/catalog/FunctionRegistry.java | 2 +-
.../java/org/apache/doris/catalog/JdbcTable.java | 2 +-
.../main/java/org/apache/doris/catalog/MTMV.java | 81 +-
.../java/org/apache/doris/catalog/OlapTable.java | 4 +-
.../cloud/datasource/CloudInternalCatalog.java | 2 +-
.../doris/cloud/system/CloudSystemInfoService.java | 8 +-
.../doris/common/cache/NereidsSqlCacheManager.java | 2 +-
.../doris/common/proc/PartitionsProcDir.java | 2 +-
.../apache/doris/common/util/ClassLoaderUtils.java | 2 +-
.../org/apache/doris/common/util/ExprUtil.java | 2 +-
.../apache/doris/datasource/CatalogProperty.java | 2 +-
.../org/apache/doris/datasource/ExternalTable.java | 2 +-
.../doris/datasource/FederationBackendPolicy.java | 2 +-
.../apache/doris/datasource/InternalCatalog.java | 2 +-
.../doris/datasource/hive/HMSExternalTable.java | 43 +-
.../apache/doris/datasource/hive/HiveDlaTable.java | 10 +-
.../doris/datasource/hive/HiveMetaStoreCache.java | 7 +-
.../doris/datasource/iceberg/IcebergUtils.java | 38 +-
.../datasource/iceberg/source/IcebergScanNode.java | 18 +-
.../doris/datasource/jdbc/JdbcExternalCatalog.java | 2 +-
.../datasource/paimon/PaimonMetadataCache.java | 2 +-
.../apache/doris/datasource/paimon/PaimonUtil.java | 2 +-
.../property/storage/HdfsProperties.java | 2 +-
.../property/storage/LocalProperties.java | 2 +-
.../doris/deploy/impl/AmbariDeployManager.java | 309 -----
.../apache/doris/deploy/impl/K8sDeployManager.java | 413 -------
.../fs/TransactionScopeCachingDirectoryLister.java | 2 +-
.../java/org/apache/doris/httpv2/HttpServer.java | 4 +-
.../httpv2/config/HttpToHttpsJettyConfig.java | 22 +-
.../apache/doris/httpv2/config/WebConfigurer.java | 31 +-
.../config/WebServerFactoryCustomizerConfig.java | 2 +-
.../doris/httpv2/controller/BaseController.java | 7 +-
.../doris/httpv2/controller/ConfigController.java | 4 +-
.../httpv2/controller/HardwareInfoController.java | 2 +-
.../doris/httpv2/controller/HelpController.java | 2 +-
.../doris/httpv2/controller/LogController.java | 2 +-
.../doris/httpv2/controller/LoginController.java | 4 +-
.../doris/httpv2/controller/LogoutController.java | 7 +-
.../doris/httpv2/controller/SessionController.java | 2 +-
.../doris/httpv2/controller/SystemController.java | 2 +-
.../doris/httpv2/interceptor/AuthInterceptor.java | 21 +-
.../httpv2/interceptor/ServletTraceIterceptor.java | 18 +-
.../doris/httpv2/meta/ColocateMetaService.java | 4 +-
.../org/apache/doris/httpv2/meta/MetaService.java | 4 +-
.../apache/doris/httpv2/rest/BackendsAction.java | 4 +-
.../doris/httpv2/rest/BootstrapFinishAction.java | 5 +-
.../apache/doris/httpv2/rest/CancelLoadAction.java | 5 +-
.../doris/httpv2/rest/CheckDecommissionAction.java | 4 +-
.../doris/httpv2/rest/CheckWalSizeAction.java | 4 +-
.../apache/doris/httpv2/rest/ConnectionAction.java | 4 +-
.../apache/doris/httpv2/rest/CopyIntoAction.java | 4 +-
.../apache/doris/httpv2/rest/DebugPointAction.java | 5 +-
.../doris/httpv2/rest/ExtraBasepathAction.java | 5 +-
.../doris/httpv2/rest/FeVersionInfoAction.java | 4 +-
.../apache/doris/httpv2/rest/GetDdlStmtAction.java | 4 +-
.../apache/doris/httpv2/rest/GetLogFileAction.java | 4 +-
.../doris/httpv2/rest/GetSmallFileAction.java | 4 +-
.../doris/httpv2/rest/GetStreamLoadState.java | 5 +-
.../org/apache/doris/httpv2/rest/HealthAction.java | 4 +-
.../org/apache/doris/httpv2/rest/LoadAction.java | 4 +-
.../apache/doris/httpv2/rest/MetaInfoAction.java | 4 +-
.../doris/httpv2/rest/MetaReplayerCheckAction.java | 4 +-
.../apache/doris/httpv2/rest/MetricsAction.java | 4 +-
.../apache/doris/httpv2/rest/MinidumpAction.java | 5 +-
.../apache/doris/httpv2/rest/ProfileAction.java | 4 +-
.../apache/doris/httpv2/rest/QueryStatsAction.java | 4 +-
.../doris/httpv2/rest/RestBaseController.java | 4 +-
.../apache/doris/httpv2/rest/RowCountAction.java | 4 +-
.../apache/doris/httpv2/rest/SetConfigAction.java | 4 +-
.../org/apache/doris/httpv2/rest/ShowAction.java | 4 +-
.../doris/httpv2/rest/StmtExecutionAction.java | 4 +-
.../doris/httpv2/rest/StorageTypeCheckAction.java | 4 +-
.../doris/httpv2/rest/TableQueryPlanAction.java | 4 +-
.../doris/httpv2/rest/TableRowCountAction.java | 4 +-
.../doris/httpv2/rest/TableSchemaAction.java | 4 +-
.../org/apache/doris/httpv2/rest/UploadAction.java | 4 +-
.../doris/httpv2/rest/manager/ClusterAction.java | 4 +-
.../doris/httpv2/rest/manager/HttpUtils.java | 2 +-
.../doris/httpv2/rest/manager/NodeAction.java | 4 +-
.../httpv2/rest/manager/QueryProfileAction.java | 4 +-
.../httpv2/restv2/AddStoragePolicyAction.java | 4 +-
.../doris/httpv2/restv2/ESCatalogAction.java | 4 +-
.../apache/doris/httpv2/restv2/ImportAction.java | 4 +-
.../doris/httpv2/restv2/MetaInfoActionV2.java | 4 +-
.../doris/httpv2/restv2/StatisticAction.java | 4 +-
.../doris/httpv2/util/StatementSubmitter.java | 2 +-
.../util/streamresponse/JsonStreamResponse.java | 2 +-
.../util/streamresponse/StreamResponseInf.java | 3 +-
.../org/apache/doris/job/base/AbstractJob.java | 2 +-
.../doris/job/executor/DispatchTaskHandler.java | 2 +-
.../doris/job/extensions/insert/InsertJob.java | 2 +-
.../insert/streaming/StreamingInsertJob.java | 2 +-
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 24 +-
.../org/apache/doris/job/manager/JobManager.java | 2 +-
.../apache/doris/job/scheduler/JobScheduler.java | 2 +-
.../org/apache/doris/load/loadv2/LoadManager.java | 2 +-
.../routineload/AbstractDataSourceProperties.java | 2 +-
.../load/routineload/KafkaRoutineLoadJob.java | 4 +-
.../doris/load/routineload/RoutineLoadManager.java | 2 +-
.../kafka/KafkaDataSourceProperties.java | 2 +-
.../org/apache/doris/master/ReportHandler.java | 2 +-
.../java/org/apache/doris/mtmv/BaseColInfo.java | 73 ++
.../org/apache/doris/mtmv/MTMVBaseVersions.java | 11 +-
.../apache/doris/mtmv/MTMVPartitionCheckUtil.java | 2 +-
.../doris/mtmv/MTMVPartitionExprDateTrunc.java | 43 +-
.../doris/mtmv/MTMVPartitionExprService.java | 2 +-
.../org/apache/doris/mtmv/MTMVPartitionInfo.java | 101 +-
.../org/apache/doris/mtmv/MTMVPartitionUtil.java | 179 +--
.../java/org/apache/doris/mtmv/MTMVPlanUtil.java | 42 +-
.../org/apache/doris/mtmv/MTMVRefreshContext.java | 8 +-
.../doris/mtmv/MTMVRefreshPartitionSnapshot.java | 16 +-
.../org/apache/doris/mtmv/MTMVRefreshSnapshot.java | 21 +-
.../MTMVRelatedPartitionDescGeneratorService.java | 5 +-
.../MTMVRelatedPartitionDescInitGenerator.java | 17 +-
...latedPartitionDescOnePartitionColGenerator.java | 27 +-
.../MTMVRelatedPartitionDescRollUpGenerator.java | 33 +-
...MTMVRelatedPartitionDescSyncLimitGenerator.java | 21 +-
.../MTMVRelatedPartitionDescTransferGenerator.java | 132 +++
.../java/org/apache/doris/mtmv/MTMVRelation.java | 2 +-
.../org/apache/doris/mtmv/MTMVRelationManager.java | 10 +-
.../org/apache/doris/mtmv/MTMVRewriteUtil.java | 63 +-
.../main/java/org/apache/doris/mtmv/MTMVUtil.java | 12 +-
.../doris/mtmv/RelatedPartitionDescResult.java | 25 +-
.../org/apache/doris/mysql/privilege/Auth.java | 2 +-
.../doris/mysql/privilege/GlobalPrivTable.java | 2 +-
.../org/apache/doris/mysql/privilege/Role.java | 2 +-
.../apache/doris/mysql/privilege/UserManager.java | 2 +-
.../doris/mysql/privilege/UserRoleManager.java | 2 +-
.../doris/nereids/analyzer/UnboundRelation.java | 2 +-
.../glue/translator/PhysicalPlanTranslator.java | 2 +-
.../nereids/load/NereidsStreamLoadPlanner.java | 2 +-
.../parser/LogicalPlanBuilderForEncryption.java | 2 +-
.../apache/doris/nereids/parser/NereidsParser.java | 2 +-
.../nereids/rules/analysis/BindExpression.java | 2 +-
.../doris/nereids/rules/analysis/BindRelation.java | 2 +-
.../nereids/rules/analysis/UserAuthentication.java | 2 +-
.../mv/AbstractMaterializedViewAggregateRule.java | 29 +-
.../mv/AbstractMaterializedViewRule.java | 80 +-
.../mv/AsyncMaterializationContext.java | 30 +
.../mv/InitMaterializationContextHook.java | 2 +-
...terializedViewAggregateOnNoneAggregateRule.java | 9 +-
.../exploration/mv/MaterializedViewUtils.java | 592 ++--------
.../rules/exploration/mv/PartitionCompensator.java | 144 ++-
.../mv/PartitionIncrementMaintainer.java | 856 ++++++++++++++
.../rules/exploration/mv/RelatedTableInfo.java | 216 ++++
.../nereids/rules/exploration/mv/StructInfo.java | 10 +-
.../expression/rules/PartitionRangeExpander.java | 2 +-
.../rules/implementation/AggregateStrategies.java | 53 +-
.../LogicalWindowToPhysicalWindow.java | 2 +-
.../rules/rewrite/PruneFileScanPartition.java | 2 +-
.../nereids/rules/rewrite/PruneOlapScanTablet.java | 2 +-
.../doris/nereids/stats/ExpressionEstimation.java | 2 +-
.../plans/commands/AdminCheckTabletsCommand.java | 2 +-
.../plans/commands/AlterRoutineLoadCommand.java | 2 +-
.../plans/commands/CreateFunctionCommand.java | 2 +-
.../commands/CreateMaterializedViewCommand.java | 3 +-
.../plans/commands/CreateTableLikeCommand.java | 2 +-
.../commands/GrantResourcePrivilegeCommand.java | 4 +-
.../plans/commands/GrantTablePrivilegeCommand.java | 4 +-
.../commands/RevokeResourcePrivilegeCommand.java | 4 +-
.../commands/RevokeTablePrivilegeCommand.java | 4 +-
.../plans/commands/ShowLoadWarningsCommand.java | 2 +-
.../plans/commands/UpdateMvByPartitionCommand.java | 65 +-
.../commands/alter/AlterRepositoryCommand.java | 2 +-
.../trees/plans/commands/info/CreateMTMVInfo.java | 17 +-
.../trees/plans/commands/info/CreateTableInfo.java | 2 +-
.../commands/info/DistributionDescriptor.java | 2 +-
.../commands/info/MTMVPartitionDefinition.java | 110 +-
.../trees/plans/commands/info/RefreshMTMVInfo.java | 10 +-
.../insert/BaseExternalTableInsertExecutor.java | 12 +-
.../commands/insert/DictionaryInsertExecutor.java | 13 +-
.../commands/insert/InsertIntoTableCommand.java | 2 +-
.../insert/InsertOverwriteTableCommand.java | 2 +-
.../trees/plans/commands/insert/InsertUtils.java | 98 +-
.../insert/OlapGroupCommitInsertExecutor.java | 13 +-
.../plans/commands/insert/OlapInsertExecutor.java | 12 +-
.../trees/plans/logical/LogicalCheckPolicy.java | 2 +-
.../nereids/trees/plans/logical/LogicalView.java | 2 +-
.../doris/nereids/util/FrontendConjunctsUtils.java | 2 +-
.../org/apache/doris/nereids/util/PlanUtils.java | 2 +-
.../doris/planner/HashDistributionPruner.java | 2 +-
.../org/apache/doris/planner/OlapScanNode.java | 2 +-
.../org/apache/doris/planner/PlanFragment.java | 2 +-
.../java/org/apache/doris/planner/PlanNode.java | 2 +-
.../java/org/apache/doris/planner/RepeatNode.java | 2 +-
.../java/org/apache/doris/planner/ScanNode.java | 2 +-
.../org/apache/doris/planner/SetOperationNode.java | 2 +-
.../java/org/apache/doris/qe/SimpleScheduler.java | 2 +-
.../org/apache/doris/qe/cache/CacheAnalyzer.java | 2 +-
.../doris/qe/runtime/ThriftPlansBuilder.java | 2 +-
.../apache/doris/service/FrontendServiceImpl.java | 2 +-
.../org/apache/doris/statistics/Histogram.java | 2 +-
.../doris/statistics/HistogramCacheLoader.java | 2 +-
.../apache/doris/statistics/StatisticsCache.java | 2 +-
.../apache/doris/statistics/StatisticsCleaner.java | 2 +-
.../doris/statistics/util/StatisticsUtil.java | 2 +-
.../java/org/apache/doris/system/HeartbeatMgr.java | 3 +-
.../doris/tablefunction/MetadataGenerator.java | 6 +-
.../PartitionsTableValuedFunction.java | 2 +-
.../org/apache/doris/task/CreateReplicaTask.java | 2 +-
.../doris/transaction/DatabaseTransactionMgr.java | 2 +-
.../doris/transaction/PublishVersionDaemon.java | 2 +-
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 2 +-
.../apache/doris/common/CacheBulkLoaderTest.java | 2 +-
.../doris/datasource/iceberg/IcebergUtilsTest.java | 8 +-
.../doris/deploy/AmbariDeployManagerTest.java | 101 --
.../apache/doris/deploy/K8sDeployManagerTest.java | 67 --
.../external/hms/MetastoreEventFactoryTest.java | 2 +-
.../doris/mtmv/MTMVPartitionCheckUtilTest.java | 8 +-
.../apache/doris/mtmv/MTMVPartitionUtilTest.java | 16 +-
.../apache/doris/mtmv/MTMVRefreshSnapshotTest.java | 21 +-
.../MTMVRelatedPartitionDescGeneratorTest.java | 295 +++++
...TMVRelatedPartitionDescRollUpGeneratorTest.java | 2 +-
.../apache/doris/mtmv/MTMVRelationManagerTest.java | 2 +-
.../org/apache/doris/mtmv/MTMVRewriteUtilTest.java | 261 ++++
.../java/org/apache/doris/mtmv/MTMVTaskTest.java | 2 +-
.../test/java/org/apache/doris/mtmv/MTMVTest.java | 4 +-
.../doris/nereids/memo/StructInfoMapTest.java | 8 +-
.../doris/nereids/mv/IdStatisticsMapTest.java | 5 +-
.../doris/nereids/mv/MvTableIdIsLongTest.java | 4 +-
.../nereids/privileges/TestCheckPrivileges.java | 2 +-
.../exploration/mv/MaterializedViewUtilsTest.java | 56 +-
.../exploration/mv/PartitionColumnTraceTest.java | 1245 ++++++++++++++++++++
.../LogicalWindowToPhysicalWindowTest.java | 2 +-
.../trees/plans/commands/BackupCommandTest.java | 2 +-
.../trees/plans/commands/RestoreCommandTest.java | 2 +-
.../plans/commands/insert/InsertUtilsTest.java | 220 ++++
.../org/apache/doris/nereids/util/PlanChecker.java | 11 +-
.../doris/planner/HashDistributionPrunerTest.java | 2 +-
.../org/apache/doris/planner/OlapScanNodeTest.java | 2 +-
.../doris/planner/QueryCacheNormalizerTest.java | 2 +-
fe/hive-udf/pom.xml | 14 +
fe/pom.xml | 198 +++-
gensrc/proto/internal_service.proto | 35 +
gensrc/script/gen_build_version.sh | 4 +-
.../json/json_group_by_and_distinct.out | 25 +
..._runtime_filter_partition_pruning_transform.out | 108 ++
.../test_ip_cidr_search_with_inverted_index.out | 9 +-
.../stream_load/load_json_with_jsonpath.out | 10 +
.../stream_load/test_load_with_jsonpath2.json | 20 +
.../data/mtmv_p0/test_hive_pct_mtmv.out | 34 +
.../data/mtmv_p0/test_multi_pct_list_mtmv.out | 24 +
.../data/mtmv_p0/test_multi_pct_mtmv.out | 55 +
.../data/mtmv_p0/test_multi_pct_nested_mtmv.out | 24 +
.../test_multi_pct_null_partition_mtmv.out} | 9 +-
.../data/mtmv_p0/test_multi_pct_rollup_mtmv.out | 28 +
.../mtmv_p0/test_multi_pct_union_list_mtmv.out | 28 +
.../data/mtmv_p0/test_multi_pct_union_mtmv.out | 28 +
.../data/mtmv_p0/test_multi_pct_view_mtmv.out | 24 +
.../data/mtmv_p0/test_null_multi_pct_mtmv.out | 27 +
.../test_upgrade_downgrade_multi_pct_mtmv.out} | 15 +-
.../multi_trace_partition_mv_rewrite.out | 41 +
.../doris/regression/action/ProfileAction.groovy | 5 +-
.../bloom_filter_p0/test_bloom_filter.groovy | 71 +-
.../cloud_p0/balance/test_balance_warm_up.groovy | 38 +-
.../test_balance_warm_up_use_peer_cache.groovy | 223 ++++
...e_warm_up_with_compaction_use_peer_cache.groovy | 234 ++++
.../read_from_peer/test_read_from_peer.groovy | 178 +++
.../cloud_p0/cache/test_topn_broadcast.groovy | 148 +++
.../test_clean_tablet_when_drop_force_table.groovy | 17 +-
.../json/json_group_by_and_distinct.groovy | 112 ++
.../fault_tolerance_nereids.groovy | 2 +-
.../manual-partitioning.md.groovy | 5 +-
...ntime_filter_partition_pruning_transform.groovy | 98 ++
.../test_iceberg_write_transform_partitions.groovy | 6 +
.../insert_overwrite_auto_detect.groovy | 6 +-
.../test_iot_overwrite_and_create.groovy | 10 +-
.../insert_p0/test_error_msg_truncate.groovy | 150 +++
.../suites/insert_p0/test_insert_docs_demo.groovy | 2 +-
...ommit_http_stream_lineitem_schema_change.groovy | 2 +
...ommit_stream_load_lineitem_schema_change.groovy | 2 +
.../test_ip_cidr_search_with_inverted_index.groovy | 4 +-
.../stream_load/load_json_with_jsonpath.groovy | 87 +-
.../test_tvf_strict_mode_and_filter_ratio.groovy | 2 +-
.../suites/mtmv_p0/test_hive_mtmv.groovy | 12 +
.../suites/mtmv_p0/test_hive_pct_mtmv.groovy | 161 +++
.../suites/mtmv_p0/test_multi_pct_bad_mtmv.groovy | 377 ++++++
.../suites/mtmv_p0/test_multi_pct_list_mtmv.groovy | 152 +++
.../suites/mtmv_p0/test_multi_pct_mtmv.groovy | 202 ++++
.../mtmv_p0/test_multi_pct_nested_mtmv.groovy | 204 ++++
.../test_multi_pct_null_partition_mtmv.groovy | 121 ++
.../mtmv_p0/test_multi_pct_rollup_mtmv.groovy | 124 ++
.../mtmv_p0/test_multi_pct_union_list_mtmv.groovy | 118 ++
.../mtmv_p0/test_multi_pct_union_mtmv.groovy | 119 ++
.../suites/mtmv_p0/test_multi_pct_view_mtmv.groovy | 159 +++
.../suites/mtmv_p0/test_null_multi_pct_mtmv.groovy | 231 ++++
.../mtmv_p0/test_partition_refresh_mtmv.groovy | 15 +-
.../suites/mtmv_up_down_multi_pct_p0/load.groovy | 76 ++
.../test_upgrade_downgrade_multi_pct_mtmv.groovy | 62 +
.../test_uniq_mv_useless.groovy | 16 +
.../explain/test_pushdown_explain.groovy | 50 +
.../cross_join_list_str_increment_create.groovy | 8 +-
.../cross_join_range_date_increment_create.groovy | 8 +-
...cross_join_range_number_increment_create.groovy | 7 +-
.../inner_join_list_str_increment_create.groovy | 14 +-
.../inner_join_range_date_increment_create.groovy | 16 +-
...inner_join_range_number_increment_create.groovy | 12 +-
...left_anti_join_list_str_increment_create.groovy | 8 +-
...ft_anti_join_range_date_increment_create.groovy | 6 +-
..._anti_join_range_number_increment_create.groovy | 9 +-
.../left_join_list_str_increment_create.groovy | 8 +-
.../left_join_range_date_increment_create.groovy | 8 +-
.../left_join_range_number_increment_create.groovy | 6 +-
...left_semi_join_list_str_increment_create.groovy | 8 +-
...ft_semi_join_range_date_increment_create.groovy | 8 +-
..._semi_join_range_number_increment_create.groovy | 9 +-
.../self_conn_list_str_increment_create.groovy | 9 +-
.../self_conn_range_date_increment_create.groovy | 8 +-
.../self_conn_range_number_increment_create.groovy | 8 +-
.../multi_trace_partition_mv_rewrite.groovy | 426 +++++++
.../test_auto_partition_behavior.groovy | 2 +-
.../test_list_partition_datatype.groovy | 2 +-
.../test_multi_column_partition.groovy | 2 +-
.../create/test_create_vault_with_kerberos.groovy | 2 +-
419 files changed, 11705 insertions(+), 2814 deletions(-)
create mode 100644 be/src/io/cache/peer_file_cache_reader.cpp
create mode 100644 be/src/io/cache/peer_file_cache_reader.h
copy fe/be-java-extensions/{avro-scanner => hadoop-deps}/pom.xml (50%)
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/deploy/impl/AmbariDeployManager.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/deploy/impl/K8sDeployManager.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/mtmv/BaseColInfo.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescTransferGenerator.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/PartitionIncrementMaintainer.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelatedTableInfo.java
delete mode 100644
fe/fe-core/src/test/java/org/apache/doris/deploy/AmbariDeployManagerTest.java
delete mode 100644
fe/fe-core/src/test/java/org/apache/doris/deploy/K8sDeployManagerTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescGeneratorTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/PartitionColumnTraceTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java
create mode 100644
regression-test/data/datatype_p0/json/json_group_by_and_distinct.out
create mode 100644
regression-test/data/load_p0/stream_load/test_load_with_jsonpath2.json
create mode 100644 regression-test/data/mtmv_p0/test_hive_pct_mtmv.out
create mode 100644 regression-test/data/mtmv_p0/test_multi_pct_list_mtmv.out
create mode 100644 regression-test/data/mtmv_p0/test_multi_pct_mtmv.out
create mode 100644 regression-test/data/mtmv_p0/test_multi_pct_nested_mtmv.out
copy regression-test/data/{load_p0/stream_load/load_json_with_jsonpath.out =>
mtmv_p0/test_multi_pct_null_partition_mtmv.out} (58%)
create mode 100644 regression-test/data/mtmv_p0/test_multi_pct_rollup_mtmv.out
create mode 100644
regression-test/data/mtmv_p0/test_multi_pct_union_list_mtmv.out
create mode 100644 regression-test/data/mtmv_p0/test_multi_pct_union_mtmv.out
create mode 100644 regression-test/data/mtmv_p0/test_multi_pct_view_mtmv.out
create mode 100644 regression-test/data/mtmv_p0/test_null_multi_pct_mtmv.out
copy regression-test/data/{load_p0/stream_load/load_json_with_jsonpath.out =>
mtmv_up_down_multi_pct_p0/test_upgrade_downgrade_multi_pct_mtmv.out} (51%)
create mode 100644
regression-test/data/nereids_rules_p0/mv/partition_union_rewrite/multi_trace_partition_mv_rewrite.out
create mode 100644
regression-test/suites/cloud_p0/balance/test_balance_warm_up_use_peer_cache.groovy
create mode 100644
regression-test/suites/cloud_p0/balance/test_balance_warm_up_with_compaction_use_peer_cache.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/read_from_peer/test_read_from_peer.groovy
create mode 100644
regression-test/suites/cloud_p0/cache/test_topn_broadcast.groovy
create mode 100644
regression-test/suites/datatype_p0/json/json_group_by_and_distinct.groovy
create mode 100644
regression-test/suites/insert_p0/test_error_msg_truncate.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_hive_pct_mtmv.groovy
create mode 100644
regression-test/suites/mtmv_p0/test_multi_pct_bad_mtmv.groovy
create mode 100644
regression-test/suites/mtmv_p0/test_multi_pct_list_mtmv.groovy
create mode 100644 regression-test/suites/mtmv_p0/test_multi_pct_mtmv.groovy
create mode 100644
regression-test/suites/mtmv_p0/test_multi_pct_nested_mtmv.groovy
create mode 100644
regression-test/suites/mtmv_p0/test_multi_pct_null_partition_mtmv.groovy
create mode 100644
regression-test/suites/mtmv_p0/test_multi_pct_rollup_mtmv.groovy
create mode 100644
regression-test/suites/mtmv_p0/test_multi_pct_union_list_mtmv.groovy
create mode 100644
regression-test/suites/mtmv_p0/test_multi_pct_union_mtmv.groovy
create mode 100644
regression-test/suites/mtmv_p0/test_multi_pct_view_mtmv.groovy
create mode 100644
regression-test/suites/mtmv_p0/test_null_multi_pct_mtmv.groovy
create mode 100644 regression-test/suites/mtmv_up_down_multi_pct_p0/load.groovy
create mode 100644
regression-test/suites/mtmv_up_down_multi_pct_p0/test_upgrade_downgrade_multi_pct_mtmv.groovy
create mode 100644
regression-test/suites/nereids_rules_p0/mv/partition_union_rewrite/multi_trace_partition_mv_rewrite.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
