This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
from 6997c87b017 branch-4.0: [bugfix](memory_profile) should split memtable
memory from task's memory profile #58908 (#58979)
add b37b8b63eb2 branch4.0: [feature](variable) enables views, materialized
views, generated columns, and alias functions to persist session variables
#58031 (#58821)
No new revisions were added by this update.
Summary of changes:
be/src/olap/memtable.cpp | 6 +-
be/src/olap/tablet_schema.cpp | 3 +-
be/src/pipeline/exec/analytic_sink_operator.h | 8 +-
be/src/runtime/primitive_type.h | 129 +----
be/src/runtime/runtime_state.h | 4 -
.../vec/aggregate_functions/aggregate_function.h | 1 -
.../aggregate_function_ai_agg.cpp | 2 +-
.../aggregate_function_approx_count_distinct.cpp | 4 +-
.../aggregate_function_array_agg.cpp | 1 +
.../aggregate_functions/aggregate_function_avg.cpp | 36 +-
.../aggregate_functions/aggregate_function_avg.h | 33 +-
.../aggregate_function_bitmap.cpp | 5 +-
.../aggregate_function_bitmap_agg.cpp | 1 +
.../aggregate_function_collect.cpp | 1 +
.../aggregate_function_corr.cpp | 2 +
.../aggregate_function_count.cpp | 5 +-
.../aggregate_function_count_by_enum.cpp | 1 +
.../aggregate_function_covar.cpp | 2 +
.../aggregate_function_distinct.cpp | 5 +-
.../aggregate_function_foreach.cpp | 10 +-
.../aggregate_function_foreachv2.cpp | 11 +-
.../aggregate_function_group_array_set_op.cpp | 8 +-
.../aggregate_function_group_concat.cpp | 1 +
.../aggregate_function_histogram.cpp | 1 +
.../aggregate_function_kurtosis.cpp | 1 +
.../aggregate_function_linear_histogram.cpp | 1 +
.../aggregate_functions/aggregate_function_map.cpp | 1 +
.../aggregate_function_map_v2.cpp | 1 +
.../aggregate_function_min_max.cpp | 7 +-
.../aggregate_function_min_max.h | 5 +-
.../aggregate_function_min_max_by.h | 1 +
.../aggregate_function_orthogonal_bitmap.cpp | 1 +
.../aggregate_function_percentile.cpp | 8 +-
.../aggregate_function_percentile_reservoir.cpp | 1 +
.../aggregate_function_product.h | 11 +-
.../aggregate_function_quantile_state.cpp | 4 +-
.../aggregate_function_quantile_state.h | 4 +-
.../aggregate_function_reader_first_last.h | 41 +-
.../aggregate_function_regr_union.cpp | 1 +
.../aggregate_functions/aggregate_function_sem.cpp | 12 +-
.../aggregate_function_sequence_match.cpp | 1 +
.../aggregate_function_simple_factory.h | 14 +-
.../aggregate_function_skew.cpp | 1 +
.../aggregate_function_stddev.cpp | 16 +-
.../aggregate_functions/aggregate_function_sum.cpp | 18 +-
.../aggregate_functions/aggregate_function_sum.h | 51 +-
.../aggregate_function_topn.cpp | 3 +
.../aggregate_function_uniq.cpp | 1 +
.../aggregate_function_uniq_distribute_key.cpp | 1 +
.../aggregate_function_window.cpp | 4 +-
.../aggregate_function_window_funnel.cpp | 1 +
be/src/vec/aggregate_functions/helpers.h | 72 +++
be/src/vec/core/call_on_type_index.h | 162 +++++++
be/src/vec/data_types/data_type_agg_state.h | 20 +-
be/src/vec/exec/scan/file_scanner.cpp | 5 +-
be/src/vec/exprs/vcast_expr.cpp | 5 +-
be/src/vec/exprs/vectorized_agg_fn.cpp | 10 +-
be/src/vec/exprs/vectorized_fn_call.cpp | 3 +-
be/src/vec/exprs/vin_predicate.cpp | 5 +-
be/src/vec/exprs/vmatch_predicate.cpp | 5 +-
be/src/vec/exprs/vtopn_pred.h | 4 +-
.../functions/array/function_array_aggregation.cpp | 491 ++++++++++++-------
.../vec/functions/array/function_array_cum_sum.cpp | 131 +++--
be/src/vec/functions/comparison_equal_for_null.cpp | 10 +-
be/src/vec/functions/function.h | 73 ++-
be/src/vec/functions/function_ifnull.h | 3 +-
be/src/vec/functions/nullif.cpp | 10 +-
be/src/vec/functions/simple_function_factory.h | 32 +-
be/test/ai/aggregate_function_ai_agg_test.cpp | 2 +-
.../operator/analytic_sink_operator_test.cpp | 35 +-
.../operator/streaming_agg_operator_test.cpp | 5 +-
be/test/testutil/mock/mock_agg_fn_evaluator.cpp | 21 +-
be/test/testutil/mock/mock_agg_fn_evaluator.h | 4 +-
.../vec/aggregate_functions/agg_array_agg_test.cpp | 11 +-
be/test/vec/aggregate_functions/agg_avg_test.cpp | 3 +-
be/test/vec/aggregate_functions/agg_bit_test.cpp | 9 +-
.../vec/aggregate_functions/agg_bitmap_test.cpp | 10 +-
.../aggregate_functions/agg_bool_union_test.cpp | 18 +-
.../vec/aggregate_functions/agg_collect_test.cpp | 14 +-
be/test/vec/aggregate_functions/agg_corr_test.cpp | 6 +-
be/test/vec/aggregate_functions/agg_count_test.cpp | 3 +-
.../vec/aggregate_functions/agg_function_test.h | 5 +-
.../agg_group_array_intersect_test.cpp | 8 +-
.../vec/aggregate_functions/agg_histogram_test.cpp | 2 +-
.../agg_linear_histogram_test.cpp | 4 +-
.../aggregate_functions/agg_min_max_by_test.cpp | 2 +-
.../vec/aggregate_functions/agg_min_max_test.cpp | 8 +-
.../vec/aggregate_functions/agg_replace_test.cpp | 4 +-
be/test/vec/aggregate_functions/agg_test.cpp | 45 +-
.../aggregate_functions/vec_count_by_enum_test.cpp | 2 +-
.../vec/aggregate_functions/vec_retention_test.cpp | 2 +-
.../vec_sequence_match_test.cpp | 10 +-
.../aggregate_functions/vec_window_funnel_test.cpp | 2 +-
.../vec/function/function_dict_get_many_test.cpp | 4 +-
be/test/vec/function/function_dict_get_test.cpp | 4 +-
.../vec/function/simple_function_factory_test.cpp | 4 +-
.../main/java/org/apache/doris/alter/Alter.java | 12 +-
.../org/apache/doris/alter/AlterJobV2Factory.java | 9 +-
.../org/apache/doris/alter/CloudRollupJobV2.java | 4 +-
.../doris/alter/MaterializedViewHandler.java | 20 +-
.../java/org/apache/doris/alter/RollupJobV2.java | 15 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 6 +-
.../org/apache/doris/analysis/MVColumnItem.java | 3 +-
.../org/apache/doris/catalog/AliasFunction.java | 27 ++
.../main/java/org/apache/doris/catalog/Column.java | 34 +-
.../java/org/apache/doris/catalog/Database.java | 7 +-
.../main/java/org/apache/doris/catalog/Env.java | 5 +-
.../org/apache/doris/catalog/FunctionUtil.java | 37 +-
.../apache/doris/catalog/GlobalFunctionMgr.java | 4 +-
.../main/java/org/apache/doris/catalog/MTMV.java | 70 ++-
.../doris/catalog/MaterializedIndexMeta.java | 108 +++--
.../java/org/apache/doris/catalog/OlapTable.java | 10 +-
.../org/apache/doris/catalog/OlapTableFactory.java | 10 +
.../main/java/org/apache/doris/catalog/View.java | 18 +-
.../main/java/org/apache/doris/mtmv/MTMVCache.java | 26 +-
.../java/org/apache/doris/mtmv/MTMVPlanUtil.java | 6 +-
.../org/apache/doris/nereids/SqlCacheContext.java | 2 +-
.../org/apache/doris/nereids/StatementContext.java | 10 +-
.../glue/translator/ExpressionTranslator.java | 6 +
.../glue/translator/PhysicalPlanTranslator.java | 46 +-
.../doris/nereids/jobs/executor/Rewriter.java | 2 +
.../apache/doris/nereids/memo/StructInfoMap.java | 13 +-
.../doris/nereids/minidump/MinidumpUtils.java | 4 +-
.../doris/nereids/parser/LogicalPlanBuilder.java | 6 +-
.../post/CommonSubExpressionCollector.java | 18 +-
.../doris/nereids/rules/analysis/BindRelation.java | 22 +-
.../doris/nereids/rules/analysis/BindSink.java | 66 ++-
.../nereids/rules/analysis/CollectRelation.java | 11 +-
.../nereids/rules/analysis/NormalizeAggregate.java | 12 +-
.../rules/analysis/SessionVarGuardRewriter.java | 149 ++++++
.../mv/InitMaterializationContextHook.java | 30 +-
.../expression/ExpressionBottomUpRewriter.java | 12 +-
.../rules/expression/ExpressionRuleType.java | 4 +-
.../nereids/rules/expression/MergeGuardExpr.java | 52 ++
.../rules/SimplifyComparisonPredicate.java | 6 +-
.../implementation/SplitAggWithoutDistinct.java | 53 ++-
.../doris/nereids/rules/rewrite/ColumnPruning.java | 2 +-
.../rules/rewrite/DistinctAggregateRewriter.java | 22 +-
.../nereids/rules/rewrite/MergeAggregate.java | 46 +-
.../doris/nereids/trees/expressions/Add.java | 2 +-
.../doris/nereids/trees/expressions/CaseWhen.java | 2 +-
.../doris/nereids/trees/expressions/Divide.java | 2 +-
.../doris/nereids/trees/expressions/Mod.java | 2 +-
.../doris/nereids/trees/expressions/Multiply.java | 2 +-
.../trees/expressions/NeedSessionVarGuard.java | 22 +-
.../trees/expressions/SessionVarGuardExpr.java | 147 ++++++
.../doris/nereids/trees/expressions/Subtract.java | 2 +-
.../trees/expressions/WindowExpression.java | 2 +-
.../functions/ComputePrecisionForSum.java | 6 +
.../functions/agg/AggregateFunction.java | 2 +-
.../trees/expressions/functions/agg/Avg.java | 4 +-
.../functions/agg/MultiDistinctSum.java | 3 +-
.../functions/agg/MultiDistinctSum0.java | 3 +-
.../trees/expressions/functions/agg/Sum.java | 3 +-
.../trees/expressions/functions/agg/Sum0.java | 3 +-
.../functions/executable/NumericArithmetic.java | 18 +-
.../expressions/functions/scalar/ArrayAvg.java | 3 +-
.../expressions/functions/scalar/ArrayCumSum.java | 4 +-
.../expressions/functions/scalar/ArrayMax.java | 3 +-
.../expressions/functions/scalar/ArrayMin.java | 3 +-
.../expressions/functions/scalar/ArrayProduct.java | 3 +-
.../expressions/functions/scalar/ArraySum.java | 3 +-
.../expressions/functions/scalar/Coalesce.java | 3 +-
.../functions/scalar/GreatestLeast.java | 3 +-
.../trees/expressions/functions/scalar/If.java | 3 +-
.../trees/expressions/functions/scalar/NullIf.java | 3 +-
.../trees/expressions/functions/scalar/Nvl.java | 3 +-
.../trees/expressions/functions/udf/AliasUdf.java | 30 +-
.../expressions/functions/udf/AliasUdfBuilder.java | 64 ++-
.../expressions/literal/DecimalV3Literal.java | 19 +-
.../nereids/trees/expressions/literal/Literal.java | 2 +-
.../expressions/visitor/ExpressionVisitor.java | 5 +
.../nereids/trees/plans/algebra/Aggregate.java | 29 ++
.../plans/commands/CreateFunctionCommand.java | 5 +-
.../commands/CreateMaterializedViewCommand.java | 5 +-
.../plans/commands/info/ColumnDefinition.java | 13 +-
.../trees/plans/commands/info/CreateMTMVInfo.java | 9 +-
.../plans/commands/info/ShowCreateMTMVInfo.java | 3 +-
.../apache/doris/nereids/types/DecimalV2Type.java | 2 +-
.../apache/doris/nereids/types/DecimalV3Type.java | 24 +-
.../org/apache/doris/nereids/util/PlanUtils.java | 42 ++
.../org/apache/doris/persist/AlterViewInfo.java | 20 +-
.../apache/doris/qe/AutoCloseSessionVariable.java | 63 +++
.../org/apache/doris/qe/ConnectContextUtil.java | 8 +
.../java/org/apache/doris/qe/SessionVariable.java | 159 ++++---
.../main/java/org/apache/doris/qe/VariableMgr.java | 87 ++--
.../doris/alter/MaterializedViewHandlerTest.java | 7 +-
.../org/apache/doris/alter/RollupJobV2Test.java | 2 +-
.../org/apache/doris/catalog/CreateViewTest.java | 5 +-
.../org/apache/doris/catalog/DatabaseTest.java | 1 +
.../expression/ExpressionRewriteTestHelper.java | 3 +-
.../doris/nereids/rules/rewrite/PrepareTest.java | 83 ++++
.../trees/expressions/VariablePersistTest.java | 215 +++++++++
.../plans/commands/merge/MergeIntoCommandTest.java | 42 +-
.../apache/doris/persist/AlterViewInfoTest.java | 3 +-
.../org/apache/doris/qe/OlapQueryCacheTest.java | 9 +-
.../org/apache/doris/qe/SessionVariablesTest.java | 3 +-
.../agg_state/decimal_sum/test_decimal_sum.out | 5 -
.../agg_state/decimalv3/test_decimalv3.out} | 16 +-
.../variables_persist/test_alias_function.out | 37 ++
.../variables_persist/test_array_agg_view.out | 41 ++
.../variables_persist/test_generated_column.out} | 18 +-
.../nereids_p0/variables_persist/test_mtmv.out | 21 +
.../nereids_p0/variables_persist/test_sync_mv.out | 16 +
.../variables_persist/test_view_var_persist.out | 216 +++++++++
.../variables_persist/use_view_create_mv.out | 11 +
.../agg_state/decimal_sum/test_decimal_sum.groovy | 31 --
.../agg_state/decimalv3/test_decimalv3.groovy | 66 +++
.../variables_persist/load.groovy} | 42 +-
.../variables_persist/test_alias_function.groovy} | 56 +--
.../variables_persist/test_array_agg_view.groovy | 77 +++
.../test_generated_column.groovy} | 71 +--
.../nereids_p0/variables_persist/test_mtmv.groovy | 249 ++++++++++
.../variables_persist/test_sync_mv.groovy | 117 +++++
.../variables_persist/test_view_var_persist.groovy | 525 +++++++++++++++++++++
.../variables_persist/use_view_create_mv.groovy | 68 +++
216 files changed, 4440 insertions(+), 1268 deletions(-)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/SessionVarGuardRewriter.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/MergeGuardExpr.java
copy
samples/arrow-flight-sql/java/src/main/java/doris/arrowflight/demo/Main.java =>
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/NeedSessionVarGuard.java
(61%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/SessionVarGuardExpr.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/qe/AutoCloseSessionVariable.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PrepareTest.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/VariablePersistTest.java
delete mode 100644
regression-test/data/datatype_p0/agg_state/decimal_sum/test_decimal_sum.out
copy
regression-test/data/{variables_up_down_decimalv3/variables_up_down_test_decimalv3.out
=> datatype_p0/agg_state/decimalv3/test_decimalv3.out} (63%)
create mode 100644
regression-test/data/nereids_p0/variables_persist/test_alias_function.out
create mode 100644
regression-test/data/nereids_p0/variables_persist/test_array_agg_view.out
copy regression-test/data/{variables_up_down_generated_column/load.out =>
nereids_p0/variables_persist/test_generated_column.out} (62%)
create mode 100644
regression-test/data/nereids_p0/variables_persist/test_mtmv.out
create mode 100644
regression-test/data/nereids_p0/variables_persist/test_sync_mv.out
create mode 100644
regression-test/data/nereids_p0/variables_persist/test_view_var_persist.out
create mode 100644
regression-test/data/nereids_p0/variables_persist/use_view_create_mv.out
delete mode 100644
regression-test/suites/datatype_p0/agg_state/decimal_sum/test_decimal_sum.groovy
create mode 100644
regression-test/suites/datatype_p0/agg_state/decimalv3/test_decimalv3.groovy
copy
regression-test/suites/{correctness/test_forbidden_infer_filter_rule.groovy =>
nereids_p0/variables_persist/load.groovy} (50%)
copy regression-test/suites/{variables_up_down_alias_function/load.groovy =>
nereids_p0/variables_persist/test_alias_function.groovy} (58%)
create mode 100644
regression-test/suites/nereids_p0/variables_persist/test_array_agg_view.groovy
copy regression-test/suites/{variables_up_down_generated_column/load.groovy =>
nereids_p0/variables_persist/test_generated_column.groovy} (72%)
create mode 100644
regression-test/suites/nereids_p0/variables_persist/test_mtmv.groovy
create mode 100644
regression-test/suites/nereids_p0/variables_persist/test_sync_mv.groovy
create mode 100644
regression-test/suites/nereids_p0/variables_persist/test_view_var_persist.groovy
create mode 100644
regression-test/suites/nereids_p0/variables_persist/use_view_create_mv.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
