This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a change to branch high-priority-column
in repository https://gitbox.apache.org/repos/asf/doris.git
omit d5fbcfd0b50 Support high priority column stats auto collection. * fix
visible column (#33023) * Collect high priority columns. (#31235) * High
priority queue and map. (#31509) * Support column level health value. (#31794)
* Support follower sync query columns to master. (#31859) * Support show auto
analyze pending jobs. (#31926) * Check column health value earlier, show job
priority. (#32064) * support window (#32094) * Refactor. (#32273) * refactor2
(#32278) * Unit test (#32398) [...]
add e82a56a1c39 [fix](nereids)EliminateGroupBy should keep the output's
datatype same as old ones (#33585)
add 3d9863f7033 [Chore](log) adjust output order on
PrintInstanceStandardInfo and reduce warning log when rpc finished (#33652)
add 93879db9beb [fix](auto bucket) Fix hit not support alter
estimate_partition_size (#33670)
add eccec4a7d39 [fix](routine-load) fix get kafka offset timeout may too
long (#33502)
add 25a85aae02d [refactor](pipelineX) Reduce prepare overhead (PART I)
(#33550)
add 4b4bb150d81 [refactor](refresh-catalog) refactor the refresh catalog
code (#33653)
add 44627042260 [fix](nereids) Use correct PREAGGREGATION in
agg(filter(scan)) (#33454)
add 0906868758b [case](regression) Add backup temp partition case (#33646)
add b0419f48d8d [fix](fe) Fix finalizeCommand `sendAndFlush`
NullPointerException (#33420)
add 3b10669ebcf [chore](cloud) Fix some comment format for critical KV
layout (#33623)
add bb288e0d294 [Enhencement](Nereids) add rule of agg(case when) to
agg(filter) (#33598)
add ef601749eac [Update](cloud) skip show data size assertion in cloud
mode (#33677)
add c6765697336 [improvement](mow) Add profile for delete_bitmap get_agg
function (#33576)
add de8c4350502 [refactor](pipelineX) Reduce prepare overhead (PART II)
(#33681)
add 71c1a8f539b [fix](testcase) fix miss used global variables in index
testcases (#33293)
add 6de9fedd49a [fix](routine-load) fix data lost when FE leader change
(#33678)
add eb8632863e2 [feature](proc)Add table's indexes info in show proc
interface (#33438)
add 1602f70d1df [fix](analyze) avoid
java.util.ConcurrentModificationException (#33674)
add 45b7eee4429 [minor](Nereids): remove useless override (#33651)
add 9e091cfc555 [Bug](runtime-filter) make need_local_merge unrelated with
broadcast and support merge on bitmap_filter (#33636)
add 67ba1e5d3f2 [opt](Nereids) date literal support basic format with
timezone (#33662)
add 9e308adf7ac [FIX](cast)fix full/right out join for cast array (#33475)
add 53f214310a9 [fix](nereids)InSubquery's withChildren method lost
typeCoercionExpr (#33692)
add 9e7204484af [improvement](spill) improve config and fix spill bugs
(#33519)
add 8d773a7aba6 [feature](agg) support aggregate function
group_array_intersect (#33265)
add c7c8916ed24 [feature](function) support hll functions hll_from_base64,
hll_to_base64 (#32089)
add c3f126bb694 [docker](hive) add hive3 docker compose and modify scripts
(#33115)
add c86d6c9b800 [fix](Nereids) could not query variant that not from table
(#33704)
add 3af3d9a07d8 [runtime filter](fix) Fix wrong results caused by
IN_OR_BLOOM filter (#33701)
add 5dfc2d5e0ad [opt](Nereids) auto fallback when meet udf override
(#33708)
add 945ee824120 (Fix)(nereids) modify create view privilege check error
message (#33669)
add 298a3489457 [improve](CI)Core modules require maintainer review
(#32468)
add e278a18f62f [fix](cloud) fix single stream multi table load can not
work in cloud mode (#33659)
add ac1ac6cb3b7 [case-wip](Vault) Add basic vault case for P0 (#33667)
add 6d441baf70c Fix use-after-free during graceful shutdown (#33457)
add d778aa53293 [fix](restore) Reset index id for restore (#33648)
add b4c7f13b26c [fix](cloud) Fix some regression case due to #33347
(#33727)
add b61369acf54 [Fix](executor)Fix routine load failed when can not find
group (#33596)
add 3e0d2759696 [opt](meta-cache) refine the meta cache (#33449)
add d8102690dba Improve analyze mv/mtmv wait row count report logic.
(#33695)
add 3117098cded [fix](parquet) fix time zone error(isAdjustedToUTC=true)
in parquet reader (#33675)
add f1d4858ccf9 [fix](move-memtable) close wait on all sinks (#33710)
add e55114dc976 [test](xor) add test for xor #33731
add 62d30734ff2 [fix](cloud)(review) map end check added (#33612) (#33613)
add 9841ac5f1fa [fix](move-memtable) ignore single replica load when move
memtable (#32845)
add 64ff8fe645a [Fix](compaction) Fix full compaction count info (#33735)
add 3e0486fa65e [fix](chore) update some CHECK to CHECK_EQ, to print more
details while CHECK failed (#33709)
add c83b8d5b3d6 [Improvement](runtime-filter) make sync rf size work when
need_local_merge (#33717)
add 5b49e8a0de9 [FIX](load)fix load with split-by-string (#33713)
add 198f8c296fb [refactor](pipelineX) Reduce prepare overhead (PART III)
(#33689)
add 6aa7665cbbf [Fix](Json type) forbit schema change adding JSON columns
with none null default value (#33686)
add 7d5a6abb9c2 [refactor](profile) refactor profile report on BE (#33331)
add b08eefefb2a [fix](test) remove array_intersect case since its result
is not stable (#33742)
add 22bfe7e3e1c [Fix](inverted index) fix memory leak when inverted index
writer not close (#33724)
add 9b2d3c242bf [fix](cloud-merge) Fix the unsteadiness cases about ttl
(#33723)
add 887ccbd5862 [feature](windows function)Improve error handling for
window functions (#33673)
add 738c199bb5b [enhancement](auditlog) ignore any errors in write audit
log (#33750)
add 0f19aaf0a56 [fix](restore) add indexes as part of table signature
(#33650)
add dbf6295c640 [chore](variable) update nereids timeout second default
value to 30 (#33749)
add bb1bbb2e576 [shuffle](minor) Log error status if exchange is shutdown
early (#33748)
add 73c336a1ae6 [improvement](spill) avoid occuping too much memory while
spill build block during the hash join build phase (#33747)
add d42b692faa8 [testcase](auto-partition) Add more details for some
testcase and enable log (#33737)
add 7408df2affe [fix](fs) Fix hdfs file writer (#33680)
add 8522d63589a [fix](inverted index) normal process query for null
condition when index is missing (#33663)
add 1e324b544f3 [fix](regression test) fix test_schema_change_fail (#33753)
add 6bbd9df2890 [fix](memory) Fix pipelinex submit task attach memory
tracker (#33732)
add d16b7ce6ee9 [fix](conf) make be conf disable_storage_page_cache
modifiable (#33773)
add 90abd76f71e [fix](Nereids) fix some sql cache consistence bug between
multiple frontends (#33722)
add de6334e3a4f [chore](Nereids)disable fallback_to_original_planner by
default (#32882)
add 5cd490121cb [minor](Nereids): remove useless stream filter() in
Translator (#33758)
add 44c3bbdf8d8 [fix](merge-cloud) fix truncate table transaction
exceeding limit (#33780)
add 7797ec9215e [fix](Nereids) dphyper support evaluate join that has one
side condition (#33702)
add 14439d265b3 [feature](Vault) Support unset default storage vault
(#33679)
add a106e45e690 Add workload group id in workload policy's property
(#33483)
add 21230a3746f [Fix](Variant Type) forbit distribution info contains
variant columns (#33707)
add 2995a78bc1d [fix](inverted index) fix incorrect case test_index_delete
(#33609)
add ce2c1ad0dee [Improvement](materialized-view) support multiple agg
function have same base table slot (#33774)
add dc80ecf7f33 [fix](Nereids) fix bug of dry run query with sql cache
(#33799)
add 1a9d7cc9df4 [fix](planner) create view statement should forbid mv
rewrite (#33784)
add 68caed86195 [Fix](Variant) VariantRootColumnIterator::read_by_rowids
with wrong null map size (#33734)
add 6dbbf2e7c8d [fix][mow] segment_creator should not flush data when
encouter abnormal exit (#33802)
add 14bbe317ad4 [Fix](Variant) check column index validation for extracted
columns (#33766)
add 7258692cb35 [Exec](join) Support column string64 to avoid join failed
in string size overflow the uint32 (#33511)
add 2f84908f5d5 [enhancement](plsql) regression for routine select and
show create procedure (#33608)
add 3d14f663a6a [opt](nereids)clear min/max column stats if table is
partially analyzed (#33685)
add c860237bf68 [exec](table_fun) opt numbers table func performance
(#33804)
add 975bf73ec21 [fix](cloud) Disable enable_feature_binlog in the cloud
mode (#33719)
add 3dd0033d4b2 [fix](Nereids): add whole tree rewriter when root is not
CTEAnchor (#33591)
add 34c57faf22f [enhancement](memory) Allocator support address sanitizers
(#33396)
add a93c86577c4 [Improvementation](join) empty_block shall be set true
when build block only one row (#33721)
add d7c472f6b1b [enhance](auth)Abstract authentication interface (#33668)
add 048bc998db0 [refactor](partitioner) refine get channel id logics
(#33765)
add 31b3feb5eae [fix](Nereids)fix unstable plan shape in limit_push_down
case
add f9e9c36bcb4 [refactor](filesystem)refactor `filesystem` interface
(#33361)
add a16d11ea610 [minor](Nereids): remove useless getFilterConjuncts()
filter() in Translator (#33801)
add 496dc3ea18c [fix](restore) Reset index id for MaterializedIndexMeta
(#33831)
add d95ac3205c1 [fix](txn insert) Fix txn insert commit failed when schema
change (#33706)
add 9d9c080bd09 [enhancement](partial-update) print more log while missed
some rowsets (#33711)
add d7b0616dbd1 [fix](compile) fix two compile errors on MacOS (#33834)
add 7876ae7dabc [fix](Regression) Don't override the regression's
enableCacheData and enableStorageVault conf (#33830)
add 9606252072d [fix](memory) temporarily close Allocator address
sanitizers #33862
add 532dc46eb8a Revert "[Fix](compaction) Fix full compaction count info
(#33735)" (#33790)
add e3ba453e9bc [feature](variable) add read_only and super_read_only
(#33795)
add 3973571bf17 [opt](nereids)do not apply CSE(Common Sub Expression) upon
multiDataSink (#33746)
add 037cf12ad5e [feature](merge-cloud) Add `BrokerLoad` and `DriverUrl`
white list (#33003)
add 8aa0b1c0959 [imporve](cloud) Get key values in batch during txn commit
(#33771)
add 64ceab497d4 [feature-wip](Cloud) Show isDefault vault in show storage
vault stmt (#33839)
add 049729a3612 [fix](move-memtable) fix commit may fail due to duplicated
reports (#32403)
add c04416b2f8d [Fix](nereids) fix bind order by expression logic (#33843)
add c6b766722be [Fix](planner) fix create view star except and modify cast
to sql (#33726)
add 401910c017f [feature](Cloud) Enable write into file cache for hdfs
writer (#33796)
add ceec5aaa35a [pipelineX](conf) Change pipeline tasks to pipelineX mode
(#33791)
add 4ebb17ff914 [bugfix](paimon) Create paimon catalog with hadoop user
(#33833)
add bbdc69930e4 [bugfix](hive)delete write path after hive insert (#33798)
add 02fa772ab6c [feature](insert)support default value when create hive
table (#33666)
add f8d8a76b46c [Fix](inverted index) fix wrong need read data opt when
encounters columnA > columnB predicate (#33855)
add c60a49b05e7 [fix](multi-table-load) fix single stream multi table load
cannot finish (#33816)
add b9952bc626b [opt](auto bucket) add fe config autobucket_max_buckets
(#33842)
add 198496bcc57 [Enhancement](group commit) Add fault injection case for
group commit
add d1a10ae6a6c [Bug](runtime-filter) fix coredump on no null string type
rf (#33869)
add 022003e840c [Improve](config)The stream_load label length is changed
to a configurable (#33745)
add 0fead9ddd70 [improvement](storage) support glibc <2.21 for system call
eventfd (#33218)
add 3740b97164c [fix](memory) Fix ExecEnv destroy memory tracking (#33781)
add 13c110863b5 [exec](table_fun) opt bitmap/split vexplode table func
performance (#33876)
add 411d146468c [fix](schema change) fix schema change check does not
calculate reader merged rows (#33825)
add 0da148ec494 [refine](node) Remove the cse DCHECK from the constructor
(#33856)
add dd470f30d65 [Fix](compile) Fix gcc compile on master (#33864)
add aadc1a6acca [fix](decompress)(review) context leaked in failure path
(#33622)
add 74ce738d367 [feature] (partition) Dynamic partition behavior changes
(#33712)
add 184a74bb939 [fix](plsql) Fix regression test for routine select
(#33860)
add 1c5d790429e [fix](stacktrace) Fix dwarf_location_info_mode is passed
as parameter to stack trace (#33863)
add b67ae877f1b [enhancement](auto-partition) Forbid use Auto and Dynamic
partition at the same time (#33736)
add 516fc4a81b6 [fix](mow) cloud full compaction should check unique key
when trying to calculate delete bitmap (#33859)
add 967ce393d2e [chore](errmsg) Fix confusing error message and clang tidy
hints (#33893)
add 3a4b7bb6b65 [feature](java-udtf) support java-udtf (#33595)
add fe251a92b1f [Chore](runtime-filter) adjust need_local_merge setting
conditions (#33886)
add b82b8dd4fec [fix](variable) modify @@auto_commit column type to BIGINT
in Nereids. #33887
add 495f3894508 [fix](memory) Fix BlockCompression memory tracking #33841
add b840c66a3c6 [opt](coord) use single instance only with small limit
(#33888)
add 07f349a3c4a [fix](stream-load) fix stream load and http stream metric
error #33899
add 91fe1e4c7c5 [fix](cloud) change preheat cache default value to fasle
(#33909)
add 3d1bdc80815 [bugfix](hive)Modify the method used to obtain the txnId
#33883
add e8a03adb7c4 [Refactor](executor)Add workload schedule policy table
(#33729)
add 68b33c878ce [feature](iceberg) add iceberg transaction implement
(#33629)
add 8a3e2f8fe2c [Fix](Csv-Reader)Fix the issue of BE core dump caused by
improper configuration of column_seperator and line_delimiter. (#33693)
add 2107bc5fdaf [fix](row-count-cache) use cached row count for show
(#33911)
add f99cef44707 [fix](nereids) do not push RF into nested cte (#33769)
add d8c4b0e6774 Revert "[Improvementation](join) empty_block shall be set
true when build block only one row (#33721)" (#33916)
add 1586cbc1bf7 [fix](cloud) Fix missing time unit ms when recycle expired
txn (#33915)
add 0fdb82c7405 [enhancement](cloud)avoid transaction too large when
dropping partition (#33700)
add 7b44e3b05b4 [fix](tablet invert index) fix tablet invert index leaky
caused by auto partition (#33714)
add 487c63842ed [refactor](opt) move BE code of hll scalar functions
together, optimize head files (#33757)
add 6d6f46e20f6 [nereids](mtmv) Support rewrite by mv nested materialized
view (#33362)
add 46975441c98 1 (#33929)
add 4d916401438 [fix](merge-cloud) fix inconsistency between cloud mode
and local mode for 2PC (#33917)
add 607a043abde [perf](routine-load) reduce waiting in task scheduler to
accelerate consume (#33756)
add 4f06106fa14 [chore](log) Avoid too many 'token parser result is empty'
(#33921)
add 06c9fb66977 [improvement](mtmv) Support union rewrite when the
materialized view is not enough to provide all the data for the query (#33800)
add eaf0157c6f5 [test](cloud) Reduce testing duration when there is no fdb
available (#33930)
add 26687556b58 Revert "[fix](tablet invert index) fix tablet invert index
leaky caused by auto partition (#33714)" (#33936)
add af589c0b13b [memtracker](accuracy) should not account resuable buffer
to query memtracker (#33933)
add 3d62ed343c0 [improvement](spill) improve spill directory and fix bugs
(#33900)
add 4d574b9bd53 [Fix](clean trash) Fix clean trash use agent task (#33912)
add 5876a326800 [fix](memory) Allocator address sanitizers enable print
stack trace (#33853)
add 84611aba86e [fix](arrow_flight_sql) Fix ArrowSchema column alias
(#33490)
add 100c09f2229 [fix](collation) add back id 33 collation for bi
compatible (#33787)
add bc701b8ebf8 [enhancement](regressiontest) fix unstable regression
tests (#33939)
add b7b7cadb6af [fix](schema change) resolve the use count check of source
logical column (#33932)
add 51f03d892c2 [fix](pipeline_x) The execution loop of a task should be
broken if the task is cancelled (#33938)
add 4b3d27d9c57 [feature](cloud) Add priority network support for
meta-service registry (#33931)
add 8738752c657 [chore](HDFS) Print file size when hdfs file writer failed
(#33918)
add 0af0a47324f Fix alter policy failed (#33910)
add 32804430d70 improve logstash doris output plugin (#33135)
add 4c4c272808a [Refator](point query) change LookupConnectionCache policy
to number (#33950)
add 440b2d00dae [feature](backend) Add cpu core count and memory usage to
"show backends" command (#33885)
add f01053ee815 [profile](scanner) Fix wrong metrics (#33965)
add aa5b4d91d0d [bug](fix) heap use after free when json parse failed
(#33955)
add 778ab9d6e9c [Enhancement](multi-catalog) Rewrite `S3URI` to remove
tricky virtual bucket mechanism and support different uri styles by flags.
(#33858)
add fe1ca68478a Adapt paimon 0.6.0 (#33943)
add f7c0d56cc07 [fix](test)fix auto partition date file not exists (#33827)
add a36a3692942 (cloud-merge) Fix the cast failed when RollupJobV2 convert
to CloudRollupJobV2 (#33847)
add 8e5369cf81d n [fix](HDFS) Correctly set hdfs cache allocator's offset
(#33960)
add 3485b7d9047 [fix](stream-load) fix query id is zero in stream load
log (#33954)
add 0f65fd0cd02 [case](Cloud) Add chaos storage vault cases (#33730)
add 59a45500374 [feat](fe) Add `ignore_bdbje_log_checksum_read` for
BDBEnvironment (#31247)
add 5fee7157ccf [fix](memory) Fix Jemalloc hook failed to start BE with
JDK 17 #33946
add 5c3a6500699 [refactor](statistic) fetch statistic data with catalog
and database id (#33928)
add d84b35ccd8c [Fix](Jdbc-Hive) fix the order of partition keys (#33963)
add e78db714760 [featrue](pipelineX) check output type in some node
(#33716)
add a7221f2a1eb [fix](profile) Fix content missing of brokerload profile
(#33969)
add c5c49686ca9 [chore](dependencies)Upgrade paimon to 0.7.0 (#33987)
add e43ed06bf7d [improve](regression test)Add test for time series compact
empty rowset (#29509)
add d5eac60734f [chore](ci) adjust conf of external pipepline (#33964)
new 18b5c3d193f Support high priority column stats auto collection. * fix
visible column (#33023) * Collect high priority columns. (#31235) * High
priority queue and map. (#31509) * Support column level health value. (#31794)
* Support follower sync query columns to master. (#31859) * Support show auto
analyze pending jobs. (#31926) * Check column health value earlier, show job
priority. (#32064) * support window (#32094) * Refactor. (#32273) * refactor2
(#32278) * Unit test (#32398) [...]
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (d5fbcfd0b50)
\
N -- N -- N refs/heads/high-priority-column (18b5c3d193f)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.clang-tidy | 2 +
.github/workflows/pr-approve-status.yml | 51 +-
be/CMakeLists.txt | 5 +
be/src/agent/agent_server.cpp | 3 +
be/src/agent/heartbeat_server.cpp | 2 +
be/src/agent/task_worker_pool.cpp | 13 +-
be/src/agent/task_worker_pool.h | 2 +
be/src/agent/workload_sched_policy_listener.cpp | 8 +-
.../cloud/cloud_engine_calc_delete_bitmap_task.cpp | 4 +-
be/src/cloud/cloud_full_compaction.cpp | 11 +-
be/src/cloud/cloud_txn_delete_bitmap_cache.cpp | 2 +-
be/src/cloud/injection_point_action.cpp | 56 +-
be/src/common/config.cpp | 20 +-
be/src/common/config.h | 18 +-
be/src/common/stack_trace.cpp | 16 +-
be/src/common/stack_trace.h | 6 +-
be/src/exec/exec_node.cpp | 22 +-
be/src/exec/rowid_fetcher.h | 8 +-
be/src/exec/schema_scanner.cpp | 50 +
be/src/exec/schema_scanner.h | 4 +
.../schema_workload_groups_scanner.cpp | 29 +-
.../schema_workload_sched_policy_scanner.cpp | 137 ++
.../schema_workload_sched_policy_scanner.h} | 38 +-
be/src/exprs/bloom_filter_func.h | 37 +-
be/src/exprs/hybrid_set.h | 78 +-
be/src/exprs/minmax_predicate.h | 41 +-
be/src/exprs/runtime_filter.cpp | 20 +-
be/src/exprs/runtime_filter_slots.h | 4 +-
be/src/glibc-compatibility/musl/eventfd.c | 8 +
be/src/http/action/http_stream.cpp | 3 +-
be/src/http/action/stream_load.cpp | 3 +-
be/src/io/cache/file_cache_common.cpp | 11 +
be/src/io/cache/file_cache_common.h | 12 +
be/src/io/file_factory.cpp | 10 +-
be/src/io/file_factory.h | 3 +-
be/src/io/fs/hdfs_file_reader.cpp | 5 +
be/src/io/fs/hdfs_file_system.cpp | 7 +-
be/src/io/fs/hdfs_file_writer.cpp | 261 +++-
be/src/io/fs/hdfs_file_writer.h | 43 +-
be/src/io/fs/multi_table_pipe.cpp | 23 +-
be/src/io/fs/stream_load_pipe.cpp | 9 +
be/src/io/fs/stream_load_pipe.h | 4 +
be/src/io/hdfs_util.cpp | 11 +
be/src/io/hdfs_util.h | 12 +
be/src/olap/comparison_predicate.h | 3 +
be/src/olap/in_list_predicate.h | 2 +
be/src/olap/like_column_predicate.h | 2 +
be/src/olap/match_predicate.h | 1 +
be/src/olap/memtable_writer.cpp | 7 +-
be/src/olap/olap_common.h | 1 +
be/src/olap/olap_server.cpp | 27 +-
be/src/olap/page_cache.h | 1 +
be/src/olap/rowset/beta_rowset_reader.cpp | 4 +
be/src/olap/rowset/beta_rowset_reader.h | 3 +
be/src/olap/rowset/beta_rowset_writer.cpp | 2 -
be/src/olap/rowset/rowset_reader.h | 2 +
be/src/olap/rowset/segment_v2/column_reader.cpp | 5 +-
be/src/olap/rowset/segment_v2/column_writer.cpp | 38 +-
.../rowset/segment_v2/hierarchical_data_reader.cpp | 2 +
.../inverted_index/query/disjunction_query.cpp | 48 +-
.../inverted_index/query/disjunction_query.h | 7 +-
.../rowset/segment_v2/inverted_index_reader.cpp | 22 +-
.../olap/rowset/segment_v2/inverted_index_reader.h | 2 +
.../rowset/segment_v2/inverted_index_writer.cpp | 7 +-
.../olap/rowset/segment_v2/inverted_index_writer.h | 22 +
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 11 +-
be/src/olap/rowset/segment_v2/segment_writer.cpp | 27 +-
.../rowset/segment_v2/vertical_segment_writer.cpp | 26 +-
be/src/olap/schema_change.cpp | 9 +-
be/src/olap/schema_change.h | 12 +-
be/src/olap/storage_engine.h | 5 +-
be/src/olap/task/engine_publish_version_task.cpp | 4 +-
be/src/olap/task/engine_publish_version_task.h | 2 +-
be/src/olap/task/index_builder.cpp | 3 +
be/src/pipeline/exec/aggregation_sink_operator.cpp | 57 +-
be/src/pipeline/exec/aggregation_sink_operator.h | 2 +
.../pipeline/exec/aggregation_source_operator.cpp | 2 +-
be/src/pipeline/exec/analytic_sink_operator.cpp | 14 +-
be/src/pipeline/exec/analytic_sink_operator.h | 1 +
be/src/pipeline/exec/analytic_source_operator.cpp | 14 +-
be/src/pipeline/exec/analytic_source_operator.h | 1 +
.../distinct_streaming_aggregation_operator.cpp | 23 +-
.../exec/distinct_streaming_aggregation_operator.h | 1 +
be/src/pipeline/exec/exchange_sink_buffer.cpp | 1 +
be/src/pipeline/exec/exchange_sink_buffer.h | 1 +
be/src/pipeline/exec/exchange_sink_operator.cpp | 51 +-
be/src/pipeline/exec/exchange_sink_operator.h | 15 +-
be/src/pipeline/exec/exchange_source_operator.cpp | 14 +-
.../exec/group_commit_block_sink_operator.cpp | 328 +++++
.../exec/group_commit_block_sink_operator.h | 82 +-
be/src/pipeline/exec/hashjoin_build_sink.cpp | 29 +-
be/src/pipeline/exec/hashjoin_probe_operator.cpp | 4 +-
be/src/pipeline/exec/join_probe_operator.cpp | 17 +-
be/src/pipeline/exec/join_probe_operator.h | 1 +
.../exec/multi_cast_data_stream_source.cpp | 19 +-
.../pipeline/exec/multi_cast_data_stream_source.h | 6 +-
.../exec/nested_loop_join_build_operator.cpp | 21 +-
.../exec/nested_loop_join_build_operator.h | 1 +
.../exec/nested_loop_join_probe_operator.cpp | 10 +-
.../exec/nested_loop_join_probe_operator.h | 1 +
be/src/pipeline/exec/olap_scan_operator.cpp | 1 +
be/src/pipeline/exec/olap_scan_operator.h | 1 +
be/src/pipeline/exec/olap_table_sink_operator.cpp | 9 -
be/src/pipeline/exec/olap_table_sink_operator.h | 7 -
.../pipeline/exec/olap_table_sink_v2_operator.cpp | 9 -
be/src/pipeline/exec/olap_table_sink_v2_operator.h | 7 -
.../exec/partitioned_aggregation_sink_operator.cpp | 9 +-
.../partitioned_aggregation_source_operator.cpp | 13 +-
.../exec/partitioned_hash_join_probe_operator.cpp | 37 +-
.../exec/partitioned_hash_join_probe_operator.h | 2 +-
.../exec/partitioned_hash_join_sink_operator.cpp | 180 ++-
.../exec/partitioned_hash_join_sink_operator.h | 4 +-
be/src/pipeline/exec/repeat_operator.cpp | 4 +-
be/src/pipeline/exec/repeat_operator.h | 2 +-
be/src/pipeline/exec/result_file_sink_operator.cpp | 4 +-
be/src/pipeline/exec/result_sink_operator.cpp | 9 +-
be/src/pipeline/exec/result_sink_operator.h | 4 +-
be/src/pipeline/exec/scan_operator.cpp | 23 +-
be/src/pipeline/exec/schema_scan_operator.cpp | 5 +-
be/src/pipeline/exec/set_probe_sink_operator.cpp | 20 +-
be/src/pipeline/exec/set_probe_sink_operator.h | 1 +
be/src/pipeline/exec/set_sink_operator.cpp | 29 +-
be/src/pipeline/exec/set_sink_operator.h | 1 +
be/src/pipeline/exec/set_source_operator.cpp | 5 +-
be/src/pipeline/exec/sort_sink_operator.cpp | 14 +-
be/src/pipeline/exec/sort_sink_operator.h | 5 +-
be/src/pipeline/exec/spill_sort_sink_operator.cpp | 34 +-
be/src/pipeline/exec/spill_sort_sink_operator.h | 2 -
.../pipeline/exec/spill_sort_source_operator.cpp | 17 +-
.../exec/streaming_aggregation_operator.cpp | 39 +-
.../pipeline/exec/streaming_aggregation_operator.h | 2 +
be/src/pipeline/exec/table_function_operator.cpp | 18 +-
be/src/pipeline/exec/table_function_operator.h | 11 +-
be/src/pipeline/exec/union_sink_operator.cpp | 13 +-
be/src/pipeline/exec/union_sink_operator.h | 1 +
be/src/pipeline/exec/union_source_operator.cpp | 17 +-
be/src/pipeline/exec/union_source_operator.h | 1 +
be/src/pipeline/pipeline_fragment_context.cpp | 2 -
be/src/pipeline/pipeline_fragment_context.h | 6 +-
.../local_exchange_sink_operator.cpp | 8 +-
.../local_exchange/local_exchange_sink_operator.h | 1 +
.../local_exchange_source_operator.cpp | 10 +-
.../local_exchange_source_operator.h | 1 +
.../pipeline_x/local_exchange/local_exchanger.cpp | 2 +-
be/src/pipeline/pipeline_x/operator.cpp | 57 +-
be/src/pipeline/pipeline_x/operator.h | 8 +-
.../pipeline_x/pipeline_x_fragment_context.cpp | 131 +-
.../pipeline_x/pipeline_x_fragment_context.h | 8 +-
be/src/pipeline/pipeline_x/pipeline_x_task.cpp | 23 +-
be/src/runtime/CMakeLists.txt | 2 +-
be/src/runtime/exec_env.h | 10 +
be/src/runtime/exec_env_init.cpp | 21 +-
be/src/runtime/fold_constant_executor.cpp | 2 +-
be/src/runtime/fragment_mgr.cpp | 282 ++--
be/src/runtime/fragment_mgr.h | 5 +
be/src/runtime/group_commit_mgr.cpp | 12 +-
be/src/runtime/jsonb_value.h | 11 +
be/src/runtime/memory/mem_tracker_limiter.cpp | 82 +-
be/src/runtime/memory/mem_tracker_limiter.h | 20 +-
be/src/runtime/plan_fragment_executor.cpp | 33 +
be/src/runtime/plan_fragment_executor.h | 4 +
be/src/runtime/primitive_type.h | 7 +-
be/src/runtime/query_context.cpp | 184 ++-
be/src/runtime/query_context.h | 69 +-
be/src/runtime/routine_load/data_consumer.cpp | 20 +-
be/src/runtime/routine_load/data_consumer.h | 4 +-
.../routine_load/routine_load_task_executor.cpp | 10 +-
.../routine_load/routine_load_task_executor.h | 6 +-
be/src/runtime/runtime_filter_mgr.cpp | 4 +-
be/src/runtime/runtime_query_statistics_mgr.cpp | 421 ++++++
be/src/runtime/runtime_query_statistics_mgr.h | 70 +
be/src/runtime/runtime_state.cpp | 7 +-
be/src/runtime/runtime_state.h | 10 +-
be/src/runtime/snapshot_loader.cpp | 1 +
.../runtime/stream_load/stream_load_executor.cpp | 6 +-
be/src/runtime/thread_context.h | 15 +-
.../workload_group/workload_group_manager.cpp | 12 +-
.../workload_management/workload_query_info.h | 1 +
.../workload_management/workload_sched_policy.cpp | 11 +-
.../workload_management/workload_sched_policy.h | 2 +
.../workload_sched_policy_mgr.cpp | 1 -
be/src/service/backend_service.cpp | 37 +-
be/src/service/backend_service.h | 7 +-
be/src/service/doris_main.cpp | 3 +
be/src/service/internal_service.cpp | 15 +-
be/src/service/point_query_executor.cpp | 16 +-
be/src/service/point_query_executor.h | 13 +-
be/src/util/arrow/row_batch.cpp | 4 +-
be/src/util/block_compression.cpp | 386 +++---
be/src/util/debug_util.cpp | 2 +-
be/src/util/easy_json.cc | 1 +
be/src/util/stack_util.cpp | 12 +-
be/src/util/stack_util.h | 6 +-
be/src/util/timezone_utils.cpp | 1 +
.../aggregate_function_group_array_intersect.cpp | 90 ++
.../aggregate_function_group_array_intersect.h | 526 ++++++++
.../aggregate_function_orthogonal_bitmap.h | 16 +-
.../aggregate_function_percentile_approx.h | 1 +
.../aggregate_function_reader_first_last.h | 4 +
.../aggregate_function_simple_factory.cpp | 2 +
.../aggregate_function_window.h | 4 +
be/src/vec/columns/column.h | 24 +-
be/src/vec/columns/column_array.cpp | 40 +-
be/src/vec/columns/column_array.h | 9 +
be/src/vec/columns/column_complex.h | 3 +
be/src/vec/columns/column_const.h | 1 +
be/src/vec/columns/column_decimal.cpp | 1 +
be/src/vec/columns/column_dictionary.h | 12 +
be/src/vec/columns/column_dummy.h | 5 +
be/src/vec/columns/column_fixed_length_object.h | 6 +
be/src/vec/columns/column_map.cpp | 37 +
be/src/vec/columns/column_map.h | 9 +
be/src/vec/columns/column_nullable.cpp | 13 +
be/src/vec/columns/column_nullable.h | 10 +-
be/src/vec/columns/column_object.h | 3 +
be/src/vec/columns/column_string.cpp | 303 +++--
be/src/vec/columns/column_string.h | 81 +-
be/src/vec/columns/column_struct.cpp | 13 +
be/src/vec/columns/column_struct.h | 12 +-
be/src/vec/columns/column_vector.cpp | 1 +
be/src/vec/columns/column_vector.h | 1 +
be/src/vec/columns/columns_common.h | 1 +
be/src/vec/columns/predicate_column.h | 15 +
be/src/vec/common/allocator.cpp | 24 +
be/src/vec/common/allocator.h | 25 +
be/src/vec/common/field_visitors.h | 1 +
be/src/vec/common/pod_array_fwd.h | 7 +-
be/src/vec/core/block.h | 31 +
be/src/vec/core/block_spill_writer.cpp | 6 +-
be/src/vec/core/decimal_comparison.h | 6 +
be/src/vec/core/field.h | 11 +
be/src/vec/core/sort_block.h | 15 +-
be/src/vec/data_types/data_type_decimal.h | 1 +
be/src/vec/data_types/data_type_nothing.cpp | 2 +
be/src/vec/data_types/data_type_struct.cpp | 3 +
be/src/vec/data_types/data_type_struct.h | 1 +
.../vec/data_types/serde/data_type_number_serde.h | 1 +
be/src/vec/exec/format/csv/csv_reader.cpp | 8 +-
be/src/vec/exec/format/json/new_json_reader.cpp | 2 +-
.../exec/format/parquet/byte_array_dict_decoder.h | 16 +-
be/src/vec/exec/format/parquet/decoder.h | 32 +-
.../exec/format/parquet/parquet_column_convert.cpp | 13 -
.../exec/format/parquet/parquet_column_convert.h | 87 +-
be/src/vec/exec/format/parquet/parquet_pred_cmp.h | 2 +-
.../format/parquet/vparquet_column_chunk_reader.h | 15 +-
.../exec/format/parquet/vparquet_column_reader.cpp | 16 +-
.../exec/format/parquet/vparquet_column_reader.h | 17 +-
be/src/vec/exec/format/table/iceberg_reader.h | 11 +-
be/src/vec/exec/join/vhash_join_node.cpp | 10 +-
be/src/vec/exec/scan/new_olap_scan_node.cpp | 145 +-
be/src/vec/exec/scan/new_olap_scan_node.h | 1 +
be/src/vec/exec/scan/new_olap_scanner.cpp | 1 +
be/src/vec/exec/scan/vscan_node.cpp | 69 +-
be/src/vec/exec/scan/vscan_node.h | 18 +-
be/src/vec/exec/scan/vscanner.cpp | 5 +-
be/src/vec/exec/vaggregation_node.cpp | 4 -
be/src/vec/exec/vaggregation_node.h | 5 -
be/src/vec/exec/vdata_gen_scan_node.cpp | 1 +
be/src/vec/exec/vexchange_node.cpp | 15 +-
be/src/vec/exec/vset_operation_node.cpp | 4 +
be/src/vec/exec/vtable_function_node.cpp | 4 +-
be/src/vec/exec/vtable_function_node.h | 11 +-
be/src/vec/exprs/table_function/table_function.h | 1 +
.../table_function/table_function_factory.cpp | 35 +-
.../exprs/table_function/table_function_factory.h | 4 +-
.../exprs/table_function/udf_table_function.cpp | 199 +++
.../vec/exprs/table_function/udf_table_function.h | 97 ++
.../vec/exprs/table_function/vexplode_bitmap.cpp | 28 +
be/src/vec/exprs/table_function/vexplode_bitmap.h | 2 +
be/src/vec/exprs/table_function/vexplode_numbers.h | 38 +-
be/src/vec/exprs/table_function/vexplode_split.cpp | 33 +-
be/src/vec/exprs/table_function/vexplode_split.h | 20 +-
be/src/vec/exprs/vectorized_agg_fn.cpp | 19 +
be/src/vec/exprs/vectorized_agg_fn.h | 4 +
be/src/vec/exprs/vectorized_fn_call.cpp | 9 +-
be/src/vec/exprs/vexpr.h | 1 +
be/src/vec/exprs/vslot_ref.cpp | 10 +-
be/src/vec/exprs/vslot_ref.h | 2 +
.../array/function_array_enumerate_uniq.cpp | 1 +
.../vec/functions/array/function_arrays_overlap.h | 13 +-
be/src/vec/functions/function.h | 1 +
be/src/vec/functions/function_cast.h | 4 +-
be/src/vec/functions/function_fake.h | 8 +
be/src/vec/functions/function_hll.cpp | 324 +++++
be/src/vec/functions/function_jsonb.cpp | 1 +
be/src/vec/functions/function_variadic_arguments.h | 1 +
be/src/vec/functions/hll_cardinality.cpp | 136 --
be/src/vec/functions/hll_empty.cpp | 43 -
be/src/vec/functions/hll_hash.cpp | 87 --
be/src/vec/functions/in.h | 28 +-
be/src/vec/functions/match.cpp | 2 +-
be/src/vec/functions/simple_function_factory.h | 8 +-
be/src/vec/json/parse2column.h | 13 +-
be/src/vec/jsonb/serialize.h | 6 +-
be/src/vec/olap/olap_data_convertor.h | 2 +
be/src/vec/runtime/partitioner.h | 17 +-
be/src/vec/runtime/vdata_stream_mgr.cpp | 21 +-
be/src/vec/runtime/vdata_stream_mgr.h | 4 +-
be/src/vec/runtime/vdata_stream_recvr.cpp | 2 +-
be/src/vec/runtime/vdatetime_value.cpp | 2 +
be/src/vec/sink/async_writer_sink.h | 23 +-
be/src/vec/sink/load_stream_map_pool.cpp | 16 +-
be/src/vec/sink/load_stream_stub.h | 12 +
be/src/vec/sink/vdata_stream_sender.cpp | 226 ++--
be/src/vec/sink/vdata_stream_sender.h | 10 +-
be/src/vec/sink/volap_table_sink.cpp | 8 +-
be/src/vec/sink/volap_table_sink.h | 4 -
be/src/vec/sink/volap_table_sink_v2.cpp | 8 +-
be/src/vec/sink/volap_table_sink_v2.h | 4 -
be/src/vec/sink/vresult_file_sink.cpp | 9 +-
be/src/vec/sink/vresult_sink.cpp | 3 +-
be/src/vec/sink/writer/vfile_result_writer.cpp | 6 +-
be/src/vec/sink/writer/vhive_partition_writer.cpp | 12 +-
be/src/vec/sink/writer/vhive_partition_writer.h | 3 +
be/src/vec/sink/writer/vtablet_writer.cpp | 8 +-
be/src/vec/sink/writer/vtablet_writer.h | 2 -
be/src/vec/sink/writer/vtablet_writer_v2.cpp | 100 +-
be/src/vec/sink/writer/vtablet_writer_v2.h | 9 +-
be/src/vec/spill/spill_stream.cpp | 5 +
be/src/vec/spill/spill_stream.h | 2 +
be/src/vec/spill/spill_stream_manager.cpp | 167 ++-
be/src/vec/spill/spill_stream_manager.h | 55 +-
be/src/vec/spill/spill_writer.cpp | 14 +-
be/test/vec/exec/parquet/parquet_thrift_test.cpp | 3 +-
be/test/vec/sink/vtablet_writer_v2_test.cpp | 239 ++++
bin/run-fs-benchmark.sh | 2 +-
bin/start_be.sh | 3 +-
build.sh | 6 +
cloud/CMakeLists.txt | 3 +
cloud/src/common/CMakeLists.txt | 3 +-
cloud/src/common/bvars.cpp | 1 +
cloud/src/common/bvars.h | 1 +
cloud/src/common/config.h | 8 +
cloud/src/common/network_util.cpp | 238 ++++
.../src/common/network_util.h | 23 +-
cloud/src/meta-service/keys.cpp | 86 +-
cloud/src/meta-service/keys.h | 60 +-
cloud/src/meta-service/meta_server.cpp | 3 +-
cloud/src/meta-service/meta_service.h | 8 +
cloud/src/meta-service/meta_service_partition.cpp | 47 +-
cloud/src/meta-service/meta_service_resource.cpp | 24 +-
cloud/src/meta-service/meta_service_txn.cpp | 359 +++--
cloud/src/meta-service/txn_kv.cpp | 87 +-
cloud/src/meta-service/txn_kv.h | 14 +-
cloud/src/recycler/recycler.cpp | 3 +-
cloud/test/CMakeLists.txt | 4 +
cloud/test/fdb_injection_test.cpp | 2 +
cloud/test/mem_txn_kv_test.cpp | 2 +
cloud/test/meta_service_test.cpp | 30 +-
cloud/test/network_util_test.cpp | 49 +
cloud/test/txn_kv_test.cpp | 2 +
...e.env.tpl.tpl => hadoop-hive-metastore.env.tpl} | 11 +-
.../{hadoop-hive.env.tpl.tpl => hadoop-hive.env} | 11 +-
.../docker-compose/hive/hive-2x.yaml.tpl | 70 +-
.../hive/{gen_env.sh.tpl => hive-2x_settings.env} | 29 +-
.../hive/{hive-2x.yaml.tpl => hive-3x.yaml.tpl} | 85 +-
.../docker-compose/hive/hive-3x_settings.env | 14 +-
.../hive/scripts/create_preinstalled_table.hql | 4 +
docker/thirdparties/run-thirdparties-docker.sh | 57 +-
extension/logstash/lib/logstash/outputs/doris.rb | 316 ++---
extension/logstash/logstash-output-doris.gemspec | 4 +-
.../java/org/apache/doris/udf/BaseExecutor.java | 4 +
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 2 +-
.../java/org/apache/doris/catalog/ScalarType.java | 14 +-
.../main/java/org/apache/doris/common/Config.java | 59 +-
.../java/org/apache/doris/common/ConfigBase.java | 12 +
.../org/apache/doris/common/FeMetaVersion.java | 4 +-
fe/fe-core/pom.xml | 5 +
fe/fe-core/src/main/cup/sql_parser.cup | 19 +-
.../src/main/java/org/apache/doris/DorisFE.java | 9 +
.../org/apache/doris/alter/AlterJobV2Factory.java | 3 +-
.../org/apache/doris/alter/CloudRollupJobV2.java | 32 +-
.../apache/doris/alter/CloudSchemaChangeJobV2.java | 36 +-
.../java/org/apache/doris/analysis/CastExpr.java | 2 +-
.../java/org/apache/doris/analysis/ColumnDef.java | 12 +-
.../apache/doris/analysis/CreateFunctionStmt.java | 35 +
.../org/apache/doris/analysis/CreateUserStmt.java | 4 +-
.../org/apache/doris/analysis/DropUserStmt.java | 4 +-
.../apache/doris/analysis/FunctionCallExpr.java | 12 +-
.../java/org/apache/doris/analysis/LoadStmt.java | 59 +
.../analysis/ModifyTablePropertiesClause.java | 2 +
.../apache/doris/analysis/NativeInsertStmt.java | 7 +-
.../org/apache/doris/analysis/PartitionDesc.java | 42 +-
.../org/apache/doris/analysis/SchemaTableType.java | 6 +-
.../java/org/apache/doris/analysis/SelectStmt.java | 7 +-
.../apache/doris/analysis/SetOperationStmt.java | 7 +
.../org/apache/doris/analysis/ShowIndexStmt.java | 1 -
.../analysis/ShowWorkloadSchedPolicyStmt.java | 59 -
.../java/org/apache/doris/analysis/SlotRef.java | 1 +
.../analysis/UnsetDefaultStorageVaultStmt.java | 22 +-
.../java/org/apache/doris/backup/Repository.java | 8 +-
.../java/org/apache/doris/backup/RestoreJob.java | 15 +-
.../apache/doris/binlog/AddPartitionRecord.java | 2 +-
.../apache/doris/catalog/AggregateFunction.java | 3 +-
.../doris/catalog/BuiltinAggregateFunctions.java | 2 +
.../doris/catalog/BuiltinScalarFunctions.java | 4 +
.../main/java/org/apache/doris/catalog/Column.java | 6 +
.../main/java/org/apache/doris/catalog/Env.java | 51 +-
.../java/org/apache/doris/catalog/EnvFactory.java | 6 +-
.../java/org/apache/doris/catalog/Function.java | 23 +-
.../org/apache/doris/catalog/FunctionRegistry.java | 9 +-
.../java/org/apache/doris/catalog/FunctionSet.java | 6 +-
.../org/apache/doris/catalog/FunctionUtil.java | 7 +-
.../doris/catalog/InternalSchemaInitializer.java | 6 +-
.../org/apache/doris/catalog/JdbcResource.java | 38 +-
.../doris/catalog/MaterializedIndexMeta.java | 4 +
.../java/org/apache/doris/catalog/OlapTable.java | 69 +-
.../java/org/apache/doris/catalog/Partition.java | 4 -
.../org/apache/doris/catalog/RefreshManager.java | 187 ++-
.../org/apache/doris/catalog/ScalarFunction.java | 15 +
.../java/org/apache/doris/catalog/SchemaTable.java | 11 +
.../org/apache/doris/catalog/StorageVault.java | 34 +-
.../org/apache/doris/catalog/StorageVaultMgr.java | 17 +
.../main/java/org/apache/doris/catalog/Table.java | 7 +-
.../java/org/apache/doris/catalog/TableIf.java | 5 +
.../doris/clone/DynamicPartitionScheduler.java | 7 +-
.../doris/cloud/catalog/CloudEnvFactory.java | 5 +-
.../apache/doris/cloud/catalog/CloudPartition.java | 5 -
.../cloud/datasource/CloudInternalCatalog.java | 17 +-
.../apache/doris/cloud/qe/CloudCoordinator.java | 5 +-
.../apache/doris/cloud/rpc/MetaServiceClient.java | 4 +
.../apache/doris/cloud/rpc/MetaServiceProxy.java | 10 +
.../transaction/CloudGlobalTransactionMgr.java | 43 +-
.../java/org/apache/doris/common/CacheFactory.java | 112 ++
.../java/org/apache/doris/common/ErrorCode.java | 2 +-
.../java/org/apache/doris/common/FeNameFormat.java | 2 +-
.../doris/common/LabelAlreadyUsedException.java | 6 +-
.../doris/common/NereidsSqlCacheManager.java | 124 +-
.../apache/doris/common/proc/BackendsProcDir.java | 9 +-
.../apache/doris/common/proc/IndexesProcNode.java | 78 ++
.../org/apache/doris/common/proc/TableProcDir.java | 6 +-
.../doris/common/profile/ExecutionProfile.java | 79 ++
.../apache/doris/common/util/AutoBucketUtils.java | 3 +-
.../org/apache/doris/common/util/BrokerUtil.java | 2 +-
.../apache/doris/common/util/CacheBulkLoader.java | 4 +-
.../doris/common/util/DynamicPartitionUtil.java | 6 +
.../apache/doris/common/util/PropertyAnalyzer.java | 9 +-
.../java/org/apache/doris/common/util/S3URI.java | 289 ++--
.../java/org/apache/doris/common/util/S3Util.java | 6 +-
.../org/apache/doris/datasource/CatalogMgr.java | 188 ---
.../apache/doris/datasource/ExternalCatalog.java | 2 +-
.../doris/datasource/ExternalMetaCacheMgr.java | 60 +-
.../apache/doris/datasource/ExternalObjectLog.java | 1 -
.../doris/datasource/ExternalRowCountCache.java | 44 +-
.../doris/datasource/ExternalSchemaCache.java | 40 +-
.../org/apache/doris/datasource/ExternalTable.java | 13 +
.../apache/doris/datasource/InternalCatalog.java | 47 +-
.../doris/datasource/hive/HMSCachedClient.java | 2 +
.../doris/datasource/hive/HMSExternalCatalog.java | 1 -
.../doris/datasource/hive/HMSExternalTable.java | 125 +-
.../doris/datasource/hive/HMSTransaction.java | 53 +-
.../doris/datasource/hive/HiveMetaStoreCache.java | 210 ++-
.../doris/datasource/hive/HiveMetadataOps.java | 2 +-
.../doris/datasource/hive/HiveVersionUtil.java | 2 +-
.../hive/PostgreSQLJdbcHMSCachedClient.java | 14 +-
.../datasource/hive/ThriftHMSCachedClient.java | 60 +-
.../datasource/hive/event/AlterPartitionEvent.java | 4 +-
.../datasource/hive/event/AlterTableEvent.java | 6 +-
.../doris/datasource/hive/event/InsertEvent.java | 6 +-
.../hive/event/MetastoreEventsProcessor.java | 2 +-
.../hudi/source/HudiCachedPartitionProcessor.java | 30 +-
.../datasource/hudi/source/HudiPartitionMgr.java | 21 +-
.../doris/datasource/hudi/source/HudiScanNode.java | 3 +-
.../datasource/iceberg/IcebergMetadataCache.java | 114 +-
.../iceberg/IcebergMetadataCacheMgr.java | 7 +-
.../datasource/iceberg/IcebergTransaction.java | 182 +++
.../doris/datasource/iceberg/IcebergUtils.java | 11 +
.../doris/datasource/iceberg/dlf/DLFCatalog.java | 9 +-
.../apache/doris/datasource/kafka/KafkaUtil.java | 9 +-
.../maxcompute/MaxComputeExternalTable.java | 40 +-
.../maxcompute/MaxComputeMetadataCache.java | 31 +-
.../maxcompute/MaxComputeMetadataCacheMgr.java | 5 +-
.../datasource/paimon/PaimonExternalCatalog.java | 16 +-
.../datasource/paimon/PaimonExternalTable.java | 4 +-
.../datasource/paimon/source/PaimonScanNode.java | 4 +-
.../datasource/property/PropertyConverter.java | 3 +
.../main/java/org/apache/doris/fs/FileSystem.java | 61 +-
.../java/org/apache/doris/fs/FileSystemCache.java | 39 +-
.../java/org/apache/doris/fs/FileSystemUtil.java | 70 +
.../org/apache/doris/fs/LocalDfsFileSystem.java | 81 +-
.../java/org/apache/doris/fs/obj/ObjStorage.java | 2 +-
.../java/org/apache/doris/fs/obj/S3ObjStorage.java | 95 +-
.../apache/doris/fs/remote/BrokerFileSystem.java | 20 +-
.../apache/doris/fs/remote/RemoteFileSystem.java | 76 +-
.../org/apache/doris/fs/remote/S3FileSystem.java | 2 +-
.../apache/doris/fs/remote/dfs/DFSFileSystem.java | 114 +-
.../org/apache/doris/httpv2/rest/ShowAction.java | 4 +-
.../apache/doris/job/extensions/mtmv/MTMVTask.java | 5 +-
.../apache/doris/journal/bdbje/BDBEnvironment.java | 5 +
.../src/main/java/org/apache/doris/load/Load.java | 6 +-
.../org/apache/doris/load/StreamLoadHandler.java | 53 +-
.../apache/doris/load/loadv2/LoadLoadingTask.java | 8 +-
.../doris/load/loadv2/LoadingTaskPlanner.java | 10 +-
.../doris/load/routineload/KafkaTaskInfo.java | 52 +-
.../doris/load/routineload/RoutineLoadJob.java | 8 +-
.../load/routineload/RoutineLoadTaskScheduler.java | 7 -
.../main/java/org/apache/doris/mtmv/MTMVCache.java | 48 +-
.../java/org/apache/doris/mysql/MysqlProto.java | 6 +-
...MysqlAuthType.java => AuthenticateRequest.java} | 36 +-
.../mysql/authenticate/AuthenticateResponse.java | 76 ++
.../{MysqlAuthType.java => AuthenticateType.java} | 4 +-
.../authenticate/Authenticator.java} | 18 +-
.../mysql/authenticate/AuthenticatorManager.java | 83 ++
.../mysql/authenticate/DefaultAuthenticator.java | 75 ++
.../apache/doris/mysql/authenticate/MysqlAuth.java | 205 ---
...dapAuthenticate.java => LdapAuthenticator.java} | 78 +-
.../doris/mysql/authenticate/ldap/LdapManager.java | 6 +-
.../mysql/authenticate/password/ClearPassword.java | 23 +-
.../password/ClearPasswordResolver.java | 60 +
.../authenticate/password/NativePassword.java} | 23 +-
.../password/NativePasswordResolver.java | 71 +
.../mysql/authenticate/password/Password.java | 17 +-
.../PasswordResolver.java} | 30 +-
.../org/apache/doris/mysql/privilege/Auth.java | 4 +-
.../org/apache/doris/nereids/NereidsPlanner.java | 82 +-
.../org/apache/doris/nereids/SqlCacheContext.java | 98 +-
.../org/apache/doris/nereids/StatementContext.java | 7 +-
.../org/apache/doris/nereids/cost/CostModelV1.java | 2 +-
.../glue/translator/ExpressionTranslator.java | 15 +-
.../glue/translator/PhysicalPlanTranslator.java | 334 +++--
.../doris/nereids/jobs/executor/Rewriter.java | 2 +
.../jobs/joinorder/hypergraph/HyperGraph.java | 4 -
.../nereids/jobs/rewrite/CostBasedRewriteJob.java | 18 +-
.../java/org/apache/doris/nereids/memo/Group.java | 20 +-
.../java/org/apache/doris/nereids/memo/Memo.java | 10 -
.../apache/doris/nereids/memo/StructInfoMap.java | 34 +-
.../doris/nereids/minidump/MinidumpUtils.java | 3 +-
.../doris/nereids/parser/PartitionTableInfo.java | 16 +-
.../processor/post/CommonSubExpressionOpt.java | 5 +-
.../processor/post/RuntimeFilterGenerator.java | 13 +-
.../org/apache/doris/nereids/rules/RuleType.java | 1 +
.../nereids/rules/analysis/BindExpression.java | 30 +-
.../nereids/rules/analysis/CheckAfterRewrite.java | 3 +-
.../nereids/rules/analysis/ExpressionAnalyzer.java | 16 +-
.../mv/AbstractMaterializedViewAggregateRule.java | 63 +-
.../mv/AbstractMaterializedViewJoinRule.java | 5 +-
.../mv/AbstractMaterializedViewRule.java | 326 +++--
.../mv/InitMaterializationContextHook.java | 27 +-
.../mv/LogicalCompatibilityContext.java | 2 +-
.../exploration/mv/MaterializationContext.java | 71 +-
.../mv/MaterializedViewAggregateRule.java | 3 +-
.../mv/MaterializedViewFilterAggregateRule.java | 10 +-
.../mv/MaterializedViewFilterJoinRule.java | 5 +-
...MaterializedViewFilterProjectAggregateRule.java | 10 +-
.../mv/MaterializedViewFilterProjectJoinRule.java | 10 +-
.../mv/MaterializedViewOnlyJoinRule.java | 5 +-
.../mv/MaterializedViewProjectAggregateRule.java | 10 +-
...MaterializedViewProjectFilterAggregateRule.java | 5 +-
.../mv/MaterializedViewProjectFilterJoinRule.java | 10 +-
.../mv/MaterializedViewProjectJoinRule.java | 10 +-
.../exploration/mv/MaterializedViewUtils.java | 101 +-
.../nereids/rules/exploration/mv/Predicates.java | 52 +
.../nereids/rules/exploration/mv/StructInfo.java | 171 ++-
.../rules/rewrite/EliminateAggCaseWhen.java | 83 ++
.../nereids/rules/rewrite/EliminateGroupBy.java | 4 +-
.../doris/nereids/rules/rewrite/EliminateSort.java | 4 +-
.../mv/AbstractSelectMaterializedIndexRule.java | 19 +-
.../mv/SelectMaterializedIndexWithAggregate.java | 189 +--
.../doris/nereids/stats/StatsCalculator.java | 56 +-
.../nereids/trees/expressions/InSubquery.java | 2 +-
.../functions/agg/GroupArrayIntersect.java | 76 ++
.../functions/scalar/HllFromBase64.java | 71 +
.../expressions/functions/scalar/HllToBase64.java | 69 +
.../trees/expressions/functions/udf/JavaUdtf.java | 171 +++
.../expressions/functions/udf/JavaUdtfBuilder.java | 87 ++
.../trees/expressions/literal/DateTimeLiteral.java | 4 +
.../visitor/AggregateFunctionVisitor.java | 5 +
.../expressions/visitor/ScalarFunctionVisitor.java | 10 +
.../visitor/TableGeneratingFunctionVisitor.java | 4 +
.../apache/doris/nereids/trees/plans/FakePlan.java | 11 +-
.../org/apache/doris/nereids/trees/plans/Plan.java | 4 +-
.../doris/nereids/trees/plans/PreAggStatus.java | 1 +
.../nereids/trees/plans/commands/Command.java | 6 +
.../plans/commands/UpdateMvByPartitionCommand.java | 32 +-
.../plans/commands/info/ColumnDefinition.java | 4 +
.../trees/plans/commands/info/CreateViewInfo.java | 19 +-
.../commands/insert/HiveInsertCommandContext.java | 18 +
.../plans/commands/insert/HiveInsertExecutor.java | 13 +-
.../commands/insert/InsertIntoTableCommand.java | 3 +-
.../trees/plans/logical/LogicalAggregate.java | 7 +
.../trees/plans/logical/LogicalOlapScan.java | 12 +
.../trees/plans/logical/LogicalSqlCache.java | 10 +-
.../plans/physical/PhysicalOlapTableSink.java | 16 -
.../trees/plans/physical/PhysicalProject.java | 5 -
.../trees/plans/physical/PhysicalSqlCache.java | 22 +-
.../plans/visitor/ExpressionLineageReplacer.java | 31 +-
.../doris/nereids/util/DateTimeFormatterUtils.java | 2 +
.../apache/doris/nereids/util/ExpressionUtils.java | 15 +-
.../org/apache/doris/nereids/util/JoinUtils.java | 17 +-
.../org/apache/doris/nereids/util/PlanUtils.java | 25 +
.../java/org/apache/doris/persist/EditLog.java | 6 +-
.../org/apache/doris/planner/HiveTableSink.java | 1 +
.../org/apache/doris/planner/OlapScanNode.java | 7 +-
.../org/apache/doris/planner/RuntimeFilter.java | 2 +-
.../java/org/apache/doris/planner/ScanNode.java | 5 +-
.../apache/doris/planner/StreamLoadPlanner.java | 34 +-
.../java/org/apache/doris/qe/AuditLogHelper.java | 16 +
.../java/org/apache/doris/qe/ConnectContext.java | 2 +-
.../java/org/apache/doris/qe/ConnectProcessor.java | 60 +-
.../main/java/org/apache/doris/qe/Coordinator.java | 34 +-
.../main/java/org/apache/doris/qe/DdlExecutor.java | 5 +-
.../java/org/apache/doris/qe/GlobalVariable.java | 24 +
.../apache/doris/qe/InsertStreamTxnExecutor.java | 153 ++-
.../java/org/apache/doris/qe/QeProcessorImpl.java | 33 +
.../java/org/apache/doris/qe/SessionVariable.java | 132 +-
.../java/org/apache/doris/qe/ShowExecutor.java | 57 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 30 +-
.../main/java/org/apache/doris/qe/VariableMgr.java | 12 +
.../org/apache/doris/qe/cache/CacheAnalyzer.java | 27 +-
.../java/org/apache/doris/qe/cache/CacheProxy.java | 1 -
.../org/apache/doris/qe/cache/RowBatchBuilder.java | 5 +-
.../java/org/apache/doris/qe/cache/SqlCache.java | 7 +-
.../resource/workloadgroup/WorkloadGroupMgr.java | 41 +-
.../workloadschedpolicy/WorkloadSchedPolicy.java | 45 +-
.../WorkloadSchedPolicyMgr.java | 50 +-
.../apache/doris/service/FrontendServiceImpl.java | 7 +-
.../apache/doris/statistics/AnalysisManager.java | 14 +-
.../doris/statistics/BasicAsyncCacheLoader.java | 4 +-
.../statistics/ColumnStatisticsCacheLoader.java | 3 +-
.../org/apache/doris/statistics/HistogramTask.java | 3 +-
.../apache/doris/statistics/StatisticsCache.java | 41 +-
.../doris/statistics/StatisticsCacheKey.java | 11 +-
.../doris/statistics/StatisticsRepository.java | 106 +-
.../doris/statistics/util/StatisticsUtil.java | 106 --
.../main/java/org/apache/doris/system/Backend.java | 12 +-
.../org/apache/doris/system/BackendHbResponse.java | 26 +
.../java/org/apache/doris/system/HeartbeatMgr.java | 3 +-
.../doris/tablefunction/MetadataGenerator.java | 20 +-
.../tablefunction/MetadataTableValuedFunction.java | 2 -
.../doris/tablefunction/S3TableValuedFunction.java | 59 +-
.../doris/tablefunction/TableValuedFunctionIf.java | 2 -
.../WorkloadSchedPolicyTableValuedFunction.java | 88 --
.../java/org/apache/doris/task/AgentBatchTask.java | 10 +
.../CleanTrashTask.java} | 24 +-
.../doris/transaction/HiveTransactionManager.java | 14 +-
.../doris/transaction/TransactionManager.java | 3 -
.../hadoop/hive/metastore/HiveMetaStoreClient.java | 5 +-
.../analysis/CreateTableAsSelectStmtTest.java | 4 +-
.../doris/analysis/ShowBackendsStmtTest.java | 2 +-
.../org/apache/doris/backup/BrokerStorageTest.java | 2 +-
.../org/apache/doris/backup/RepositoryTest.java | 8 +-
.../apache/doris/catalog/CreateFunctionTest.java | 8 +-
.../apache/doris/catalog/RefreshCatalogTest.java | 142 --
.../apache/doris/common/CacheBulkLoaderTest.java | 20 +-
.../org/apache/doris/common/CacheFactoryTest.java | 291 ++++
.../org/apache/doris/common/GenericPoolTest.java | 13 +-
.../doris/common/proc/IndexesProcNodeTest.java | 102 ++
.../org/apache/doris/common/util/S3URITest.java | 153 ++-
.../apache/doris/datasource/CatalogMgrTest.java | 7 +-
.../datasource/ExternalRowCountCacheTest.java | 100 --
.../doris/datasource/RefreshCatalogTest.java | 94 +-
...hedClientTest.java => TestHMSCachedClient.java} | 7 +-
.../doris/datasource/hive/HmsCommitTest.java | 177 ++-
.../datasource/iceberg/IcebergTransactionTest.java | 328 +++++
.../datasource/property/PropertyConverterTest.java | 4 +-
.../org/apache/doris/fs/obj/S3FileSystemTest.java | 8 +-
.../org/apache/doris/fs/obj/S3ObjStorageTest.java | 10 +-
.../org/apache/doris/mysql/MysqlProtoTest.java | 53 +-
.../authenticate/DefaultAuthenticatorTest.java | 100 ++
.../authenticate/ldap/LdapAuthenticateTest.java | 203 ---
.../authenticate/ldap/LdapAuthenticatorTest.java | 146 ++
.../joinorder/hypergraph/CompareOuterJoinTest.java | 29 +-
.../jobs/joinorder/hypergraph/InferJoinTest.java | 21 +-
.../joinorder/hypergraph/InferPredicateTest.java | 10 +-
.../jobs/joinorder/hypergraph/OtherJoinTest.java | 20 +-
.../joinorder/hypergraph/PullupExpressionTest.java | 22 +-
.../joinorder/joinhint/DistributeHintTest.java | 4 +-
.../org/apache/doris/nereids/memo/RankTest.java | 6 +-
.../doris/nereids/memo/StructInfoMapTest.java | 2 +-
.../rules/exploration/mv/BuildStructInfoTest.java | 10 +-
.../rules/exploration/mv/EliminateJoinTest.java | 30 +-
.../rules/exploration/mv/HyperGraphAggTest.java | 11 +-
.../exploration/mv/HyperGraphComparatorTest.java | 22 +-
.../nereids/rules/rewrite/EliminateFilterTest.java | 42 +-
.../rules/rewrite/EliminateGroupByTest.java | 2 +-
.../rules/rewrite/mv/SelectRollupIndexTest.java | 34 +-
.../doris/nereids/sqltest/JoinOrderJobTest.java | 4 +-
.../expressions/literal/DateTimeLiteralTest.java | 16 +-
.../doris/nereids/util/ExpressionUtilsTest.java | 8 +-
.../doris/nereids/util/HyperGraphBuilder.java | 16 +-
.../doris/planner/ListPartitionPrunerV2Test.java | 6 +-
.../doris/planner/TableFunctionPlanTest.java | 2 +-
.../org/apache/doris/qe/HmsQueryCacheTest.java | 48 +-
.../java/org/apache/doris/qe/VariableMgrTest.java | 19 +
.../org/apache/doris/service/FlightSqlJDBC.java | 1 -
.../org/apache/doris/statistics/CacheTest.java | 4 +-
.../doris/utframe/DemoMultiBackendsTest.java | 8 +-
.../apache/doris/utframe/MockedBackendFactory.java | 13 +-
fe/pom.xml | 9 +-
gensrc/proto/cloud.proto | 17 +
gensrc/proto/internal_service.proto | 1 +
gensrc/script/doris_builtins_functions.py | 7 +-
gensrc/thrift/AgentService.thrift | 3 +
gensrc/thrift/BackendService.thrift | 16 +-
gensrc/thrift/DataSinks.thrift | 15 +
gensrc/thrift/Descriptors.thrift | 3 +-
gensrc/thrift/Exprs.thrift | 1 +
gensrc/thrift/FrontendService.thrift | 26 +-
gensrc/thrift/HeartbeatService.thrift | 2 +-
gensrc/thrift/Types.thrift | 2 +
regression-test/conf/regression-conf.groovy | 4 +-
.../test_time_series_compaction_policy.out | 10 +
.../data/correctness_p0/test_bit_function.out | 17 +
.../export/hive_read/orc/test_hive_read_orc.out | 3 -
.../orc/test_hive_read_orc_complex_type.out | 2 +-
.../hive_read/parquet/test_hive_read_parquet.out | 3 -
.../hive/test_hive_basic_type.out | 24 +-
.../jdbc/test_mariadb_jdbc_catalog.out | 1 +
.../jdbc/test_mysql_jdbc_catalog.out | 1 +
.../jdbc/test_mysql_jdbc_catalog_nereids.out | 1 +
.../jdbc/test_mysql_jdbc_driver5_catalog.out | 1 +
.../tvf/test_s3_tvf_parquet_compress.out | 120 +-
.../hive/test_external_catalog_hive.out | 17 +-
.../hive/test_hive_parquet_alter_column.out | 114 +-
.../test_no_index_null_fault_injection.out} | 0
.../txn_insert_values_with_schema_change.out | 13 +
.../test_index_no_need_read_data.out | 74 ++
.../data/javaudf_p0/test_javaudtf_all_types.out | 1390 ++++++++++++++++++++
.../data/javaudf_p0/test_javaudtf_arrayint.out | 25 +
.../data/javaudf_p0/test_javaudtf_decimal.out | 15 +
.../data/javaudf_p0/test_javaudtf_double.out | 15 +
.../data/javaudf_p0/test_javaudtf_int.out | 32 +
.../data/javaudf_p0/test_javaudtf_string.out | 34 +
.../stream_load/load_json_parse_failed/test | 4 +
.../data/load_p0/stream_load/special_seperator.csv | 2 +
.../stream_load/test_array_split_string.csv | 1 +
.../stream_load/test_array_with_func_load.out | 4 +
.../stream_load/test_csv_special_seperator.out | 5 +
.../load_p0/stream_load/test_stream_load_2pc.out | 5 +
.../multi_agg_with_same_slot.out | 48 +
.../data/mv_p0/mv_percentile/mv_percentile.out | 36 +
.../data/nereids_function_p0/agg_function/agg.out | 142 ++
.../data/nereids_function_p0/scalar_function/H.out | 116 ++
.../explain/test_scan_preaggregation_explain.out | 7 +
.../eliminate_aggregate_casewhen.out | 249 ++++
.../limit_push_down/limit_push_down.out | 89 +-
.../limit_push_down/order_push_down.out | 86 +-
.../mv/nested/nested_materialized_view.out | 20 +
.../nereids_rules_p0/mv/partition_mv_rewrite.out | 29 +
.../nereids_syntax_p0/order_by_bind_priority.out | 22 +
.../data/nereids_syntax_p0/runtime_filter.out | 39 +
.../runtime_filter/test_pushdown_setop.out | 35 +-
.../data/plsql_p0/test_plsql_routine.out | 38 +
.../query_p0/aggregate/group_array_intersect.out | 93 ++
...nnel_account_consume_history_reconciliation.csv | 200 +++
.../data/query_p0/join/dwd_oa_customer.csv | 200 +++
.../join/no_null_str_rf/no_null_str_rf.out | 4 +
regression-test/data/query_p0/join/tb_media.csv | 10 +
.../query_p0/join/tb_reconciliation_detail.csv | 200 +++
.../join/tb_reconciliation_media_order.csv | 200 +++
.../query_p0/join/test_join_with_cast_array.out | 194 +++
.../hll_functions/test_hll_functions.out | 35 +
.../data/variable_p0/set_and_unset_variable.out | 24 +
.../data/variant_p0/sql/select_from_value.out | 4 +
.../data/variant_p0/topn_opt_read_by_rowids.out | 14 +
.../data/variant_p0/with_index/var_index.out | 12 +
.../create_view_star_except_and_cast_to_sql.out | 11 +
.../test_workload_sched_policy.out | 17 +-
.../org/apache/doris/regression/Config.groovy | 12 +-
.../apache/doris/regression/ConfigOptions.groovy | 9 +
.../org/apache/doris/regression/suite/Suite.groovy | 25 +
.../java/org/apache/doris/udf/UDTFAllTypeTest.java | 210 +++
.../org/apache/doris/udf/UDTFArrayIntTest.java | 22 +-
.../java/org/apache/doris/udf/UDTFDecimalTest.java | 22 +-
.../java/org/apache/doris/udf/UDTFDoubleTest.java | 24 +-
.../java/org/apache/doris/udf/UDTFIntTest.java | 25 +-
.../java/org/apache/doris/udf/UDTFNullTest.java | 21 +-
.../java/org/apache/doris/udf/UDTFStringTest.java | 22 +-
.../pipeline/cloud_p0/conf/be_custom.conf | 5 +-
.../pipeline/cloud_p0/conf/fe_custom.conf | 2 +
.../cloud_p0/conf/regression-conf-custom.groovy | 2 +
.../pipeline/cloud_p1/conf/be_custom.conf | 5 +-
.../pipeline/cloud_p1/conf/fe_custom.conf | 2 +
.../cloud_p1/conf/regression-conf-custom.groovy | 2 +
regression-test/pipeline/common/doris-utils.sh | 2 +-
.../pipeline/external/conf/regression-conf.groovy | 56 +-
regression-test/pipeline/performance/compile.sh | 1 +
.../suites/autobucket/test_autobucket.groovy | 33 +-
.../backup_restore/test_backup_restore.groovy | 9 +-
...st_backup_restore_backup_temp_partition.groovy} | 58 +-
.../test_backup_restore_overwrite_indexes.groovy | 172 +++
.../auth/test_grant_revoke_cluster_role.groovy | 38 +-
.../auth/test_grant_revoke_cluster_to_user.groovy | 28 +-
.../suites/cloud_p0/cache/ttl/alter_ttl_1.groovy | 1 +
.../suites/cloud_p0/cache/ttl/alter_ttl_2.groovy | 21 +
.../suites/cloud_p0/cache/ttl/alter_ttl_4.groovy | 13 +-
.../suites/cloud_p0/cache/ttl/test_ttl.groovy | 16 +-
...compaction_uniq_cluster_keys_with_delete.groovy | 2 +-
.../test_time_series_compaction_policy.groovy | 161 +++
.../suites/correctness_p0/test_bit_function.groovy | 9 +
.../suites/correctness_p0/test_mv_case.groovy | 32 +
.../test_window_function_error.groovy | 47 +
.../decimalv3/test_decimal256_index.groovy | 3 +
.../export/hive_read/orc/test_hive_read_orc.groovy | 14 +-
.../orc/test_hive_read_orc_complex_type.groovy | 8 +-
.../parquet/test_hive_read_parquet.groovy | 14 +-
.../test_hive_read_parquet_comlex_type.groovy | 8 +-
.../hive/ddl/test_hive_ctas.groovy | 47 +-
.../hive/ddl/test_hive_ddl.groovy | 261 ++--
.../hive/ddl/test_hive_write_type.groovy | 1 +
.../jdbc/test_mysql_jdbc_catalog.groovy | 30 +-
.../jdbc/test_mysql_jdbc_driver5_catalog.groovy | 30 +-
.../paimon/paimon_timestamp_types.groovy | 5 +-
.../test_admin_clean_trash.groovy | 687 ++++++++++
.../test_commit_info_fault_injection.groovy | 99 ++
...oup_commit_async_wal_msg_fault_injection.groovy | 40 +-
.../test_no_index_null_fault_injection.groovy | 65 +
.../suites/hdfs_vault/default_vault_p2/load.groovy | 4 +-
.../multi_vault_p2/ddl/customer_create.sql | 2 +-
.../hdfs_vault/multi_vault_p2/ddl/date_create.sql | 2 +-
.../multi_vault_p2/ddl/lineorder_create.sql | 2 +-
.../multi_vault_p2/ddl/lineorder_flat_create.sql | 2 +-
.../hdfs_vault/multi_vault_p2/ddl/part_create.sql | 2 +-
.../multi_vault_p2/ddl/supplier_create.sql | 2 +-
.../suites/hdfs_vault/multi_vault_p2/load.groovy | 4 +-
.../hdfs_vault/ssb_sf1_p2/ddl/customer_create.sql | 2 +-
.../hdfs_vault/ssb_sf1_p2/ddl/date_create.sql | 2 +-
.../hdfs_vault/ssb_sf1_p2/ddl/lineorder_create.sql | 2 +-
.../ssb_sf1_p2/ddl/lineorder_flat_create.sql | 2 +-
.../hdfs_vault/ssb_sf1_p2/ddl/part_create.sql | 2 +-
.../hdfs_vault/ssb_sf1_p2/ddl/supplier_create.sql | 2 +-
.../suites/hdfs_vault/ssb_sf1_p2/load.groovy | 2 +-
.../inject_hdfs_load_error.groovy | 116 ++
.../inject_hdfs_select_error.groovy | 104 ++
.../txn_insert_values_with_schema_change.groovy | 109 ++
.../test_index_no_need_read_data.groovy | 871 +++++++++++-
.../suites/inverted_index_p0/test_show_data.groovy | 48 +-
.../javaudf_p0/test_javaudtf_all_types.groovy | 233 ++++
.../javaudf_p0/test_javaudtf_arrayint.groovy | 75 ++
.../suites/javaudf_p0/test_javaudtf_decimal.groovy | 67 +
.../suites/javaudf_p0/test_javaudtf_double.groovy | 70 +
.../suites/javaudf_p0/test_javaudtf_int.groovy | 74 ++
.../suites/javaudf_p0/test_javaudtf_string.groovy | 85 ++
.../load_json_parse_failed.groovy | 49 +
.../stream_load/test_array_with_func_load.groovy | 56 +
.../stream_load/test_csv_special_seperator.groovy | 69 +
.../load_p0/stream_load/test_pipeline_load.groovy | 2 +-
.../load_p0/stream_load/test_stream_load.groovy | 2 +-
.../stream_load/test_stream_load_2pc.groovy | 150 +++
.../test_stream_load_move_memtable.groovy | 9 +
.../suites/load_p0/tvf/test_s3_tvf.groovy | 86 +-
.../mtmv_p0/test_hive_multi_partition_mtmv.groovy | 6 +
.../suites/mtmv_p0/test_hive_refresh_mtmv.groovy | 8 +
.../multi_agg_with_same_slot.groovy | 73 +
.../mv_p0/mv_percentile/mv_percentile.groovy | 66 +
.../nereids_function_p0/agg_function/agg.groovy | 67 +
.../nereids_function_p0/scalar_function/H.groovy | 8 +
.../nereids_p0/aggregate/agg_window_project.groovy | 3 +
.../cache/parse_sql_from_sql_cache.groovy | 163 ++-
.../test_scan_preaggregation_explain.groovy | 49 +
.../nereids_p0/subquery/test_subquery.groovy | 1 +
.../suites/nereids_p0/test_mv_select.groovy | 4 +
.../eliminate_aggregate_casewhen.groovy | 108 ++
.../limit_push_down/limit_push_down.groovy | 8 +-
.../limit_push_down/order_push_down.groovy | 48 +-
.../agg_with_roll_up/aggregate_with_roll_up.groovy | 56 +-
.../mv/dimension/dimension_self_conn.groovy | 564 ++++++++
.../mv_contain_external_table.groovy | 4 +
.../mv/join/inner/inner_join.groovy | 20 +-
.../mv/nested/nested_materialized_view.groovy | 182 +++
.../mv/nested_mtmv/nested_mtmv.groovy | 859 ++++++++++++
.../mv/partition_mv_rewrite.groovy | 17 +
.../partition_curd_union_rewrite.groovy} | 85 +-
.../mv/union_rewrite/usercase_union_rewrite.groovy | 174 +++
.../order_by_bind_priority.groovy | 36 +
.../suites/nereids_syntax_p0/runtime_filter.groovy | 69 +
.../test_auto_partition_behavior.groovy | 53 +-
.../test_auto_range_partition.groovy | 4 +
.../doris_dbgen_conf/two_stream_load_conflict.yaml | 4 +
.../auto_partition/sql/multi_thread_load.groovy | 22 +-
.../test_dynamic_partition.groovy | 73 +
.../suites/plsql_p0/test_plsql_routine.groovy | 68 +
.../aggregate/group_array_intersect.groovy | 106 ++
.../join/no_null_str_rf/no_null_str_rf.groovy | 58 +
.../query_p0/join/test_join_with_cast_array.groovy | 291 ++++
.../hll_functions/test_hll_functions.groovy | 72 +
.../window_functions/test_ntile_function.groovy | 2 +-
.../multi_vault_p2/ddl/customer_create.sql | 2 +-
.../s3_vault/multi_vault_p2/ddl/date_create.sql | 2 +-
.../multi_vault_p2/ddl/lineorder_create.sql | 2 +-
.../multi_vault_p2/ddl/lineorder_flat_create.sql | 2 +-
.../s3_vault/multi_vault_p2/ddl/part_create.sql | 2 +-
.../multi_vault_p2/ddl/supplier_create.sql | 2 +-
.../s3_vault/ssb_sf1_p2/ddl/customer_create.sql | 2 +-
.../suites/s3_vault/ssb_sf1_p2/ddl/date_create.sql | 2 +-
.../s3_vault/ssb_sf1_p2/ddl/lineorder_create.sql | 2 +-
.../ssb_sf1_p2/ddl/lineorder_flat_create.sql | 2 +-
.../suites/s3_vault/ssb_sf1_p2/ddl/part_create.sql | 2 +-
.../s3_vault/ssb_sf1_p2/ddl/supplier_create.sql | 2 +-
.../test_unique_model_schema_key_change.groovy | 14 +-
.../test_schema_change_fail.groovy | 4 +-
.../suites/statistics/test_analyze_mtmv.groovy | 11 +-
.../suites/statistics/test_analyze_mv.groovy | 14 +-
.../suites/tpch_sf0.1_p1/sql/cse.groovy | 10 +
.../variable_p0/set_and_unset_variable.groovy | 22 +
.../suites/variant_github_events_p0/load.groovy | 108 +-
regression-test/suites/variant_p0/load.groovy | 12 +
.../suites/variant_p0/sql/select_from_value.sql | 1 +
.../variant_p0/topn_opt_read_by_rowids.groovy | 104 ++
.../suites/variant_p0/with_index/load.groovy | 5 +-
.../suites/variant_p0/with_index/var_index.groovy | 8 +-
regression-test/suites/vaults/create/create.groovy | 148 +++
.../suites/vaults/default/default.groovy | 108 ++
.../create_view_star_except_and_cast_to_sql.groovy | 45 +
.../workload_manager_p0/test_curd_wlg.groovy | 6 +-
.../test_workload_sched_policy.groovy | 103 +-
run-be-ut.sh | 3 +-
tools/maintainers/check_review.py | 96 ++
tools/maintainers/maintainers.json | 12 +
909 files changed, 27533 insertions(+), 7516 deletions(-)
create mode 100644
be/src/exec/schema_scanner/schema_workload_sched_policy_scanner.cpp
copy be/src/{vec/exprs/table_function/table_function_factory.h =>
exec/schema_scanner/schema_workload_sched_policy_scanner.h} (54%)
create mode 100644 be/src/pipeline/exec/group_commit_block_sink_operator.cpp
create mode 100644
be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.cpp
create mode 100644
be/src/vec/aggregate_functions/aggregate_function_group_array_intersect.h
create mode 100644 be/src/vec/exprs/table_function/udf_table_function.cpp
create mode 100644 be/src/vec/exprs/table_function/udf_table_function.h
create mode 100644 be/src/vec/functions/function_hll.cpp
delete mode 100644 be/src/vec/functions/hll_cardinality.cpp
delete mode 100644 be/src/vec/functions/hll_empty.cpp
delete mode 100644 be/src/vec/functions/hll_hash.cpp
create mode 100644 be/test/vec/sink/vtablet_writer_v2_test.cpp
create mode 100644 cloud/src/common/network_util.cpp
copy be/src/runtime/workload_management/workload_query_info.h =>
cloud/src/common/network_util.h (66%)
create mode 100644 cloud/test/network_util_test.cpp
copy docker/thirdparties/docker-compose/hive/{hadoop-hive.env.tpl.tpl =>
hadoop-hive-metastore.env.tpl} (90%)
rename docker/thirdparties/docker-compose/hive/{hadoop-hive.env.tpl.tpl =>
hadoop-hive.env} (90%)
rename docker/thirdparties/docker-compose/hive/{gen_env.sh.tpl =>
hive-2x_settings.env} (50%)
mode change 100755 => 100644
copy docker/thirdparties/docker-compose/hive/{hive-2x.yaml.tpl =>
hive-3x.yaml.tpl} (54%)
copy
regression-test/suites/partition_p1/auto_partition/doris_dbgen_conf/two_stream_load_conflict.yaml
=> docker/thirdparties/docker-compose/hive/hive-3x_settings.env (65%)
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/analysis/ShowWorkloadSchedPolicyStmt.java
copy be/src/runtime/workload_management/workload_query_info.h =>
fe/fe-core/src/main/java/org/apache/doris/analysis/UnsetDefaultStorageVaultStmt.java
(74%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexesProcNode.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemUtil.java
copy
fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/{MysqlAuthType.java
=> AuthenticateRequest.java} (59%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/AuthenticateResponse.java
copy
fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/{MysqlAuthType.java
=> AuthenticateType.java} (93%)
copy
fe/fe-core/src/main/java/org/apache/doris/{transaction/TransactionManager.java
=> mysql/authenticate/Authenticator.java} (69%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/AuthenticatorManager.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/DefaultAuthenticator.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/MysqlAuth.java
rename
fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/ldap/{LdapAuthenticate.java
=> LdapAuthenticator.java} (57%)
copy be/src/runtime/workload_management/workload_query_info.h =>
fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/password/ClearPassword.java
(74%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/password/ClearPasswordResolver.java
copy
fe/fe-core/src/main/java/org/apache/doris/{nereids/trees/plans/commands/insert/HiveInsertCommandContext.java
=> mysql/authenticate/password/NativePassword.java} (63%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/password/NativePasswordResolver.java
copy be/src/runtime/workload_management/workload_query_info.h =>
fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/password/Password.java
(74%)
rename
fe/fe-core/src/main/java/org/apache/doris/mysql/authenticate/{MysqlAuthType.java
=> password/PasswordResolver.java} (56%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateAggCaseWhen.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/agg/GroupArrayIntersect.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HllFromBase64.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HllToBase64.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtf.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/udf/JavaUdtfBuilder.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/tablefunction/WorkloadSchedPolicyTableValuedFunction.java
copy
fe/fe-core/src/main/java/org/apache/doris/{nereids/trees/plans/commands/insert/HiveInsertCommandContext.java
=> task/CleanTrashTask.java} (60%)
delete mode 100644
fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshCatalogTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/common/CacheFactoryTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/common/proc/IndexesProcNodeTest.java
delete mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java
rename
fe/fe-core/src/test/java/org/apache/doris/datasource/{HMSCachedClientTest.java
=> TestHMSCachedClient.java} (98%)
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/mysql/authenticate/DefaultAuthenticatorTest.java
delete mode 100644
fe/fe-core/src/test/java/org/apache/doris/mysql/authenticate/ldap/LdapAuthenticateTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/mysql/authenticate/ldap/LdapAuthenticatorTest.java
create mode 100644
regression-test/data/compaction/test_time_series_compaction_policy.out
copy regression-test/data/{inverted_index_p0/test_compound.out =>
fault_injection_p0/test_no_index_null_fault_injection.out} (100%)
create mode 100644
regression-test/data/insert_p0/txn_insert_values_with_schema_change.out
create mode 100644 regression-test/data/javaudf_p0/test_javaudtf_all_types.out
create mode 100644 regression-test/data/javaudf_p0/test_javaudtf_arrayint.out
create mode 100644 regression-test/data/javaudf_p0/test_javaudtf_decimal.out
create mode 100644 regression-test/data/javaudf_p0/test_javaudtf_double.out
create mode 100644 regression-test/data/javaudf_p0/test_javaudtf_int.out
create mode 100644 regression-test/data/javaudf_p0/test_javaudtf_string.out
create mode 100644
regression-test/data/load_p0/stream_load/load_json_parse_failed/test
create mode 100644
regression-test/data/load_p0/stream_load/special_seperator.csv
create mode 100644
regression-test/data/load_p0/stream_load/test_array_split_string.csv
create mode 100644
regression-test/data/load_p0/stream_load/test_array_with_func_load.out
create mode 100644
regression-test/data/load_p0/stream_load/test_csv_special_seperator.out
create mode 100644
regression-test/data/load_p0/stream_load/test_stream_load_2pc.out
create mode 100644
regression-test/data/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.out
create mode 100644 regression-test/data/mv_p0/mv_percentile/mv_percentile.out
create mode 100644
regression-test/data/nereids_p0/explain/test_scan_preaggregation_explain.out
create mode 100644
regression-test/data/nereids_rules_p0/eliminate_aggregate_casewhen/eliminate_aggregate_casewhen.out
create mode 100644
regression-test/data/nereids_rules_p0/mv/nested/nested_materialized_view.out
create mode 100644
regression-test/data/nereids_rules_p0/mv/partition_mv_rewrite.out
create mode 100644 regression-test/data/nereids_syntax_p0/runtime_filter.out
create mode 100644 regression-test/data/plsql_p0/test_plsql_routine.out
create mode 100644
regression-test/data/query_p0/aggregate/group_array_intersect.out
create mode 100644
regression-test/data/query_p0/join/dwd_channel_account_consume_history_reconciliation.csv
create mode 100644 regression-test/data/query_p0/join/dwd_oa_customer.csv
create mode 100644
regression-test/data/query_p0/join/no_null_str_rf/no_null_str_rf.out
create mode 100644 regression-test/data/query_p0/join/tb_media.csv
create mode 100644
regression-test/data/query_p0/join/tb_reconciliation_detail.csv
create mode 100644
regression-test/data/query_p0/join/tb_reconciliation_media_order.csv
create mode 100644
regression-test/data/query_p0/join/test_join_with_cast_array.out
create mode 100644
regression-test/data/query_p0/sql_functions/hll_functions/test_hll_functions.out
create mode 100644 regression-test/data/variant_p0/sql/select_from_value.out
create mode 100644 regression-test/data/variant_p0/topn_opt_read_by_rowids.out
create mode 100644
regression-test/data/view_p0/create_view_star_except_and_cast_to_sql.out
create mode 100644
regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFAllTypeTest.java
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertCommandContext.java
=>
regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFArrayIntTest.java
(68%)
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertCommandContext.java
=>
regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDecimalTest.java
(69%)
copy be/src/runtime/workload_management/workload_query_info.h =>
regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFDoubleTest.java
(73%)
copy be/src/runtime/workload_management/workload_query_info.h =>
regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFIntTest.java
(72%)
copy be/src/runtime/workload_management/workload_query_info.h =>
regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFNullTest.java
(74%)
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertCommandContext.java
=>
regression-test/java-udf-src/src/main/java/org/apache/doris/udf/UDTFStringTest.java
(69%)
copy regression-test/suites/backup_restore/{test_backup_restore.groovy =>
test_backup_restore_backup_temp_partition.groovy} (62%)
create mode 100644
regression-test/suites/backup_restore/test_backup_restore_overwrite_indexes.groovy
create mode 100644
regression-test/suites/compaction/test_time_series_compaction_policy.groovy
create mode 100644
regression-test/suites/correctness_p0/test_window_function_error.groovy
create mode 100644
regression-test/suites/fault_injection_p0/test_admin_clean_trash.groovy
create mode 100644
regression-test/suites/fault_injection_p0/test_commit_info_fault_injection.groovy
create mode 100644
regression-test/suites/fault_injection_p0/test_no_index_null_fault_injection.groovy
create mode 100755
regression-test/suites/inject_hdfs_vault_p0/inject_hdfs_load_error.groovy
create mode 100755
regression-test/suites/inject_hdfs_vault_p0/inject_hdfs_select_error.groovy
create mode 100644
regression-test/suites/insert_p0/txn_insert_values_with_schema_change.groovy
create mode 100644
regression-test/suites/javaudf_p0/test_javaudtf_all_types.groovy
create mode 100644
regression-test/suites/javaudf_p0/test_javaudtf_arrayint.groovy
create mode 100644
regression-test/suites/javaudf_p0/test_javaudtf_decimal.groovy
create mode 100644
regression-test/suites/javaudf_p0/test_javaudtf_double.groovy
create mode 100644 regression-test/suites/javaudf_p0/test_javaudtf_int.groovy
create mode 100644
regression-test/suites/javaudf_p0/test_javaudtf_string.groovy
create mode 100644
regression-test/suites/load_p0/stream_load/load_json_parse_failed/load_json_parse_failed.groovy
create mode 100644
regression-test/suites/load_p0/stream_load/test_array_with_func_load.groovy
create mode 100644
regression-test/suites/load_p0/stream_load/test_csv_special_seperator.groovy
create mode 100644
regression-test/suites/load_p0/stream_load/test_stream_load_2pc.groovy
create mode 100644
regression-test/suites/mv_p0/multi_agg_with_same_slot/multi_agg_with_same_slot.groovy
create mode 100644
regression-test/suites/mv_p0/mv_percentile/mv_percentile.groovy
create mode 100644
regression-test/suites/nereids_p0/explain/test_scan_preaggregation_explain.groovy
create mode 100644
regression-test/suites/nereids_rules_p0/eliminate_aggregate_casewhen/eliminate_aggregate_casewhen.groovy
create mode 100644
regression-test/suites/nereids_rules_p0/mv/dimension/dimension_self_conn.groovy
create mode 100644
regression-test/suites/nereids_rules_p0/mv/nested/nested_materialized_view.groovy
create mode 100644
regression-test/suites/nereids_rules_p0/mv/nested_mtmv/nested_mtmv.groovy
copy regression-test/suites/nereids_rules_p0/mv/{partition_mv_rewrite.groovy
=> union_rewrite/partition_curd_union_rewrite.groovy} (66%)
create mode 100644
regression-test/suites/nereids_rules_p0/mv/union_rewrite/usercase_union_rewrite.groovy
create mode 100644
regression-test/suites/partition_p1/dynamic_partition/test_dynamic_partition.groovy
create mode 100644 regression-test/suites/plsql_p0/test_plsql_routine.groovy
create mode 100644
regression-test/suites/query_p0/aggregate/group_array_intersect.groovy
create mode 100644
regression-test/suites/query_p0/join/no_null_str_rf/no_null_str_rf.groovy
create mode 100644
regression-test/suites/query_p0/join/test_join_with_cast_array.groovy
create mode 100644
regression-test/suites/query_p0/sql_functions/hll_functions/test_hll_functions.groovy
create mode 100644 regression-test/suites/variant_p0/sql/select_from_value.sql
create mode 100644
regression-test/suites/variant_p0/topn_opt_read_by_rowids.groovy
create mode 100644 regression-test/suites/vaults/create/create.groovy
create mode 100644 regression-test/suites/vaults/default/default.groovy
create mode 100644
regression-test/suites/view_p0/create_view_star_except_and_cast_to_sql.groovy
create mode 100644 tools/maintainers/check_review.py
create mode 100644 tools/maintainers/maintainers.json
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
