This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
from a1db5c6f52 [fix](vec) crash caused by not-implemented function in
ColumnFixedLengthObject (#17215)
add 9db56201a6 [refactor](Nereids) Refactor rewrite framework to speed up
plan (#17126)
No new revisions were added by this update.
Summary of changes:
.../org/apache/doris/nereids/CascadesContext.java | 93 ++++++-
.../org/apache/doris/nereids/NereidsPlanner.java | 21 +-
.../nereids/{jobs/JobType.java => PlanSource.java} | 19 +-
.../doris/nereids/analyzer/NereidsAnalyzer.java | 79 ++++--
.../org/apache/doris/nereids/analyzer/Scope.java | 9 +-
.../java/org/apache/doris/nereids/jobs/Job.java | 42 +--
.../org/apache/doris/nereids/jobs/JobContext.java | 13 +-
.../org/apache/doris/nereids/jobs/JobType.java | 3 +-
.../nereids/jobs/{JobType.java => RewriteJob.java} | 20 +-
.../TopicRewriteJob.java} | 45 ++--
.../AdjustAggregateNullableForEmptySetJob.java | 34 ---
.../doris/nereids/jobs/batch/AnalyzeRulesJob.java | 82 ------
...ConvertApplyToJoinJob.java => ApplyToJoin.java} | 27 +-
.../doris/nereids/jobs/batch/BatchRewriteJob.java | 116 ++++++++
.../doris/nereids/jobs/batch/BatchRulesJob.java | 109 --------
...ubqueryRulesJob.java => CascadesOptimizer.java} | 24 +-
.../doris/nereids/jobs/batch/CheckAnalysisJob.java | 35 ---
....java => CorrelateApplyToUnCorrelateApply.java} | 41 +--
...ob.java => EliminateUselessPlanUnderApply.java} | 25 +-
.../jobs/batch/NereidsRewriteJobExecutor.java | 150 -----------
.../doris/nereids/jobs/batch/NereidsRewriter.java | 214 +++++++++++++++
.../doris/nereids/jobs/batch/OptimizeRulesJob.java | 34 ---
.../nereids/jobs/rewrite/CustomRewriteJob.java | 78 ++++++
.../jobs/rewrite/PlanTreeRewriteBottomUpJob.java | 128 +++++++++
.../nereids/jobs/rewrite/PlanTreeRewriteJob.java | 117 ++++++++
.../jobs/rewrite/PlanTreeRewriteTopDownJob.java | 66 +++++
.../nereids/jobs/rewrite/RewriteJobContext.java | 65 +++++
.../jobs/rewrite/RootPlanTreeRewriteJob.java | 171 ++++++++++++
.../nereids/jobs/rewrite/VisitorRewriteJob.java | 69 -----
.../doris/nereids/jobs/scheduler/JobScheduler.java | 8 +-
.../{JobScheduler.java => ScheduleContext.java} | 13 +-
.../nereids/jobs/scheduler/SimpleJobScheduler.java | 11 +-
.../java/org/apache/doris/nereids/memo/Group.java | 13 +-
.../java/org/apache/doris/nereids/memo/Memo.java | 4 +-
.../doris/nereids/pattern/MatchingContext.java | 4 +
.../pattern/{Patterns.java => MemoPatterns.java} | 25 +-
.../org/apache/doris/nereids/pattern/Pattern.java | 39 ++-
.../org/apache/doris/nereids/pattern/Patterns.java | 298 ---------------------
.../pattern/{Patterns.java => PlanPatterns.java} | 81 ++----
.../ProxyPattern.java} | 34 +--
.../generator/LogicalBinaryPatternGenerator.java | 10 +-
.../generator/LogicalLeafPatternGenerator.java | 4 +-
.../generator/LogicalUnaryPatternGenerator.java | 10 +-
.../generator/PatternDescribableProcessor.java | 36 ++-
.../pattern/generator/PatternGenerator.java | 49 ++--
.../generator/PatternGeneratorAnalyzer.java | 10 +-
.../generator/PhysicalBinaryPatternGenerator.java | 10 +-
.../generator/PhysicalLeafPatternGenerator.java | 4 +-
.../generator/PhysicalUnaryPatternGenerator.java | 10 +-
.../doris/nereids/processor/post/Validator.java | 6 +-
.../doris/nereids/rules/AppliedAwareRule.java | 111 ++++++++
...liminateLimitUnderApply.java => ProxyRule.java} | 38 +--
.../java/org/apache/doris/nereids/rules/Rule.java | 5 +
.../apache/doris/nereids/rules/RuleFactory.java | 4 +-
.../org/apache/doris/nereids/rules/RuleType.java | 17 +-
.../rules/analysis/AnalysisRuleFactory.java | 3 +-
.../rules/analysis/AvgDistinctToSumDivCount.java | 3 +-
.../nereids/rules/analysis/BindExpression.java | 138 ++++++----
.../doris/nereids/rules/analysis/BindRelation.java | 54 ++--
.../doris/nereids/rules/analysis/CheckPolicy.java | 18 +-
.../doris/nereids/rules/analysis/RegisterCTE.java | 10 +-
.../analysis/ReplaceExpressionByChildOutput.java | 8 +-
.../ResolveOrdinalInOrderByAndGroupBy.java | 15 +-
.../doris/nereids/rules/analysis/SlotBinder.java | 11 +-
.../nereids/rules/analysis/SubExprAnalyzer.java | 15 +-
.../{AnalyzeSubquery.java => SubqueryToApply.java} | 29 +-
.../nereids/rules/analysis/UserAuthentication.java | 4 +-
.../rules/exploration/ExplorationRuleFactory.java | 3 +-
.../rewrite/ExpressionNormalization.java | 11 +-
.../expression/rewrite/ExpressionRewrite.java | 82 ++++--
.../rewrite/ExpressionRewriteContext.java | 5 +-
.../expression/rewrite/ExpressionRuleExecutor.java | 24 +-
.../expression/rewrite/rules/FoldConstantRule.java | 7 -
.../rules/implementation/AggregateStrategies.java | 30 ++-
.../implementation/ImplementationRuleFactory.java | 3 +-
.../mv/SelectMaterializedIndexWithAggregate.java | 13 +-
.../SelectMaterializedIndexWithoutAggregate.java | 31 ++-
...leFactory.java => BatchRewriteRuleFactory.java} | 24 +-
.../nereids/rules/rewrite/RewriteRuleFactory.java | 3 +-
.../CheckAndStandardizeWindowFunctionAndFrame.java | 4 +-
.../rules/rewrite/logical/EliminateAggregate.java | 4 +-
.../rewrite/logical/EliminateGroupByConstant.java | 9 +-
.../rewrite/logical/EliminateLimitUnderApply.java | 2 +-
.../rules/rewrite/logical/EliminateNotNull.java | 8 +-
.../rules/rewrite/logical/EliminateOuterJoin.java | 4 +-
.../rewrite/logical/EliminateSortUnderApply.java | 6 +-
.../logical/EliminateUnnecessaryProject.java | 123 +++++----
.../logical/ExtractFilterFromCrossJoin.java | 4 +-
...xtractSingleTableExpressionFromDisjunction.java | 1 -
.../rewrite/logical/FindHashConditionForJoin.java | 4 +
.../logical/HideOneRowRelationUnderUnion.java | 4 +-
.../rules/rewrite/logical/InferFilterNotNull.java | 7 +-
.../rules/rewrite/logical/InferJoinNotNull.java | 15 +-
.../rules/rewrite/logical/InferPredicates.java | 8 +-
.../rules/rewrite/logical/MergeFilters.java | 4 -
.../rules/rewrite/logical/MergeGenerates.java | 4 +-
.../rules/rewrite/logical/MergeProjects.java | 4 +-
.../rules/rewrite/logical/MergeSetOperations.java | 65 +++--
.../rewrite/logical/PruneAggChildColumns.java | 4 +-
.../rewrite/logical/PruneFilterChildColumns.java | 5 +-
.../rewrite/logical/PruneJoinChildrenColumns.java | 5 +-
.../rewrite/logical/PruneSortChildColumns.java | 5 +-
...orrelatedFilterUnderApplyAggregateProject.java} | 18 +-
...erProject.java => PullUpProjectUnderApply.java} | 16 +-
.../rewrite/logical/PushFilterInsideJoin.java | 9 +-
.../logical/PushdownFilterThroughAggregation.java | 3 +-
.../rewrite/logical/PushdownFilterThroughJoin.java | 10 +-
.../logical/PushdownFilterThroughProject.java | 53 +++-
.../logical/PushdownFilterThroughRepeat.java | 3 +-
.../logical/PushdownProjectThroughLimit.java | 10 +-
.../nereids/rules/rewrite/logical/ReorderJoin.java | 13 +-
....java => UnCorrelatedApplyAggregateFilter.java} | 12 +-
...derFilter.java => UnCorrelatedApplyFilter.java} | 9 +-
...ct.java => UnCorrelatedApplyProjectFilter.java} | 11 +-
.../org/apache/doris/nereids/trees/TreeNode.java | 17 ++
.../doris/nereids/trees/expressions/Exists.java | 8 +-
.../nereids/trees/expressions/Expression.java | 10 +-
.../nereids/trees/expressions/InSubquery.java | 8 +-
.../doris/nereids/trees/expressions/IsNull.java | 2 +-
.../nereids/trees/expressions/SlotReference.java | 3 +-
.../nereids/trees/expressions/SubqueryExpr.java | 40 +--
.../doris/nereids/trees/plans/AbstractPlan.java | 27 +-
.../apache/doris/nereids/trees/plans/FakePlan.java | 13 +
.../org/apache/doris/nereids/trees/plans/Plan.java | 16 ++
.../apache/doris/nereids/trees/plans/PlanType.java | 6 +-
.../nereids/trees/plans/commands/Command.java | 49 ++--
.../trees/plans/commands/CreatePolicyCommand.java | 4 +-
.../trees/plans/commands/ExplainCommand.java | 4 +-
.../trees/plans/logical/AbstractLogicalPlan.java | 11 +-
.../trees/plans/logical/LogicalAggregate.java | 2 +-
.../nereids/trees/plans/logical/LogicalLeaf.java | 2 +-
.../trees/plans/logical/LogicalOlapScan.java | 3 +
.../trees/plans/logical/LogicalProject.java | 4 +-
.../nereids/trees/plans/logical/LogicalRepeat.java | 3 +-
.../trees/plans/logical/LogicalSetOperation.java | 16 +-
.../plans/logical/OutputSavePoint.java} | 19 +-
.../trees/plans/visitor/CustomRewriter.java} | 15 +-
.../apache/doris/nereids/util/ExpressionUtils.java | 14 +-
.../apache/doris/nereids/util/MutableState.java | 92 +++++++
.../java/org/apache/doris/nereids/util/Utils.java | 1 +
.../org/apache/doris/nereids/JoinHintTest.java | 4 +-
.../nereids/datasets/ssb/SSBJoinReorderTest.java | 4 +-
.../org/apache/doris/nereids/memo/MemoTest.java | 4 +-
.../apache/doris/nereids/metrics/EventTest.java | 5 +
.../doris/nereids/parser/ParserTestBase.java | 4 +-
.../pattern/GroupExpressionMatchingTest.java | 2 +-
.../rules/analysis/AnalyzeSubQueryTest.java | 20 +-
.../rules/analysis/AnalyzeWhereSubqueryTest.java | 62 +++--
.../nereids/rules/analysis/BindFunctionTest.java | 4 +-
.../analysis/CheckExpressionLegalityTest.java | 4 +-
.../rules/analysis/FillUpMissingSlotsTest.java | 4 +-
.../rules/analysis/FunctionRegistryTest.java | 4 +-
.../rules/analysis/NormalizeRepeatTest.java | 4 +-
.../nereids/rules/analysis/RegisterCTETest.java | 12 +-
.../ReplaceExpressionByChildOutputTest.java | 4 +-
.../join/InnerJoinLAsscomProjectTest.java | 4 +-
.../exploration/join/InnerJoinLAsscomTest.java | 4 +-
.../join/InnerJoinLeftAssociateTest.java | 4 +-
.../join/InnerJoinRightAssociateTest.java | 4 +-
.../rules/exploration/join/JoinCommuteTest.java | 4 +-
.../rules/exploration/join/JoinExchangeTest.java | 4 +-
.../rules/exploration/join/OuterJoinAssocTest.java | 19 +-
.../join/OuterJoinLAsscomProjectTest.java | 4 +-
.../exploration/join/OuterJoinLAsscomTest.java | 4 +-
.../join/SemiJoinSemiJoinTransposeProjectTest.java | 4 +-
.../rewrite/ExpressionRewriteTestHelper.java | 21 +-
.../rules/expression/rewrite/FoldConstantTest.java | 2 +-
.../expression/rewrite/SimplifyRangeTest.java | 13 +-
.../doris/nereids/rules/mv/SelectMvIndexTest.java | 4 +-
.../nereids/rules/mv/SelectRollupIndexTest.java | 4 +-
.../rewrite/logical/AggregateStrategiesTest.java | 4 +-
.../CheckAndStandardizeWindowFunctionTest.java | 4 +-
.../rules/rewrite/logical/ColumnPruningTest.java | 4 +-
.../logical/EliminateDedupJoinConditionTest.java | 4 +-
.../rewrite/logical/EliminateOuterJoinTest.java | 16 +-
.../logical/EliminateUnnecessaryProjectTest.java | 22 +-
.../ExtractAndNormalizeWindowExpressionTest.java | 4 +-
.../logical/ExtractFilterFromCrossJoinTest.java | 4 +-
...ctSingleTableExpressionFromDisjunctionTest.java | 4 +-
.../rewrite/logical/InferFilterNotNullTest.java | 4 +-
.../rewrite/logical/InferJoinNotNullTest.java | 12 +-
.../rules/rewrite/logical/InferPredicatesTest.java | 4 +-
.../rules/rewrite/logical/LimitPushDownTest.java | 4 +-
.../logical/LogicalWindowToPhysicalWindowTest.java | 4 +-
.../rules/rewrite/logical/MergeProjectsTest.java | 4 +-
.../rewrite/logical/NormalizeAggregateTest.java | 4 +-
.../logical/PhysicalStorageLayerAggregateTest.java | 4 +-
.../rewrite/logical/PushFilterInsideJoinTest.java | 4 +-
.../PushdownExpressionsInHashConditionTest.java | 4 +-
.../PushdownFilterThroughAggregationTest.java | 8 +-
.../logical/PushdownFilterThroughJoinTest.java | 4 +-
.../logical/PushdownProjectThroughLimitTest.java | 4 +-
.../rules/rewrite/logical/ReorderJoinTest.java | 4 +-
.../apache/doris/nereids/sqltest/InferTest.java | 2 +-
.../apache/doris/nereids/sqltest/SqlTestBase.java | 4 +-
.../trees/expressions/SelectExceptTest.java | 4 +-
.../doris/nereids/trees/expressions/ViewTest.java | 4 +-
.../nereids/trees/plans/PlanToStringTest.java | 2 +-
.../apache/doris/nereids/util/MatchingUtils.java | 12 +-
...pported.java => MemoPatternMatchSupported.java} | 4 +-
.../apache/doris/nereids/util/MemoTestUtils.java | 5 +-
.../org/apache/doris/nereids/util/PlanChecker.java | 16 +-
...pported.java => PlanPatternMatchSupported.java} | 5 +-
.../apache/doris/utframe/TestWithFeService.java | 11 +-
.../explain_clickbench_benchmark.groovy | 3 +-
205 files changed, 2714 insertions(+), 1914 deletions(-)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/{jobs/JobType.java =>
PlanSource.java} (71%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/{JobType.java =>
RewriteJob.java} (75%)
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/{rules/rewrite/logical/EliminateLimitUnderApply.java
=> jobs/TopicRewriteJob.java} (51%)
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AdjustAggregateNullableForEmptySetJob.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/AnalyzeRulesJob.java
rename
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/{ConvertApplyToJoinJob.java
=> ApplyToJoin.java} (70%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRewriteJob.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/BatchRulesJob.java
rename
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/{AnalyzeSubqueryRulesJob.java
=> CascadesOptimizer.java} (58%)
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/CheckAnalysisJob.java
rename
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/{AdjustApplyFromCorrelateToUnCorrelateJob.java
=> CorrelateApplyToUnCorrelateApply.java} (54%)
rename
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/{EliminateSpecificPlanUnderApplyJob.java
=> EliminateUselessPlanUnderApply.java} (72%)
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/OptimizeRulesJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CustomRewriteJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RewriteJobContext.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/VisitorRewriteJob.java
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/scheduler/{JobScheduler.java
=> ScheduleContext.java} (73%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/{Patterns.java
=> MemoPatterns.java} (92%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/{Patterns.java
=> PlanPatterns.java} (79%)
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/{jobs/scheduler/SimpleJobScheduler.java
=> pattern/ProxyPattern.java} (56%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/AppliedAwareRule.java
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/{rewrite/logical/EliminateLimitUnderApply.java
=> ProxyRule.java} (52%)
rename
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/{AnalyzeSubquery.java
=> SubqueryToApply.java} (88%)
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/{RewriteRuleFactory.java
=> BatchRewriteRuleFactory.java} (62%)
rename
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/{ApplyPullFilterOnProjectUnderAgg.java
=> PullUpCorrelatedFilterUnderApplyAggregateProject.java} (82%)
rename
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/{PushApplyUnderProject.java
=> PullUpProjectUnderApply.java} (82%)
rename
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/{ApplyPullFilterOnAgg.java
=> UnCorrelatedApplyAggregateFilter.java} (89%)
rename
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/{PushApplyUnderFilter.java
=> UnCorrelatedApplyFilter.java} (89%)
rename
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/{EliminateFilterUnderApplyProject.java
=> UnCorrelatedApplyProjectFilter.java} (90%)
copy fe/fe-core/src/main/java/org/apache/doris/nereids/{jobs/JobType.java =>
trees/plans/logical/OutputSavePoint.java} (71%)
copy
fe/fe-core/src/{test/java/org/apache/doris/nereids/util/PatternMatchSupported.java
=> main/java/org/apache/doris/nereids/trees/plans/visitor/CustomRewriter.java}
(71%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/util/MutableState.java
copy
fe/fe-core/src/test/java/org/apache/doris/nereids/util/{PatternMatchSupported.java
=> MemoPatternMatchSupported.java} (87%)
rename
fe/fe-core/src/test/java/org/apache/doris/nereids/util/{PatternMatchSupported.java
=> PlanPatternMatchSupported.java} (87%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
