This is an automated email from the ASF dual-hosted git repository.
starocean999 pushed a change to branch dev_rec4
in repository https://gitbox.apache.org/repos/asf/doris.git
from 21beaee3c1c support rec cte (be part / proto part)
add 7a033b84314 [Fix](workflow) Fix Mac be-ut github action (#58914)
add 961fc6abff8 [fix](regr) Use Youngs-Cramer for REGR_SLOPE/INTERCEPT to
align with PG (#55940)
add 000532eeb6a [opt](hive) use binary search to prune hive partitions
(#58877)
add 8ff0adeda1b [fix](test) Fix output result for
test_non_overlap_seg_heavy_sc (#58805)
add 2badeef0cfc [Refactor](cast) Make all nullable for cast from string to
simple types (#57949)
add 19cbb4b1a92 [Enhancement](insert-overwrite) Make clean garbage
partition interval of InsertOverwriteManager adjustable (#58733)
add 21aad940990 [fix](close) Release resource once fragment was cancelled
(#58909)
add 157afed7024 [feature](inverted index) Add inverted index NORMALIZER
support (#58178)
add 884611d3588 [fix](test) fix regression case limit_push_down (#58873)
add 4ab82f7ab72 [test](doris catalog)Add test cases for doris catalog
(#58831)
add c95b8f9debb [check](column) add column self check (#58706)
add 8e3860351a4 [chore](contrib) update faiss commit (#58900)
add 7d9385061ab recursive cte fe part
add ed721f8e10e recursive cte be part
add a0a1f264e51 remove useless case
add ddc111fd90e rebase code
No new revisions were added by this update.
Summary of changes:
.github/workflows/be-ut-mac.yml | 2 +-
be/src/olap/inverted_index_parser.cpp | 16 +-
be/src/olap/inverted_index_parser.h | 8 +-
be/src/olap/rowset/segment_v2/column_reader.cpp | 16 +-
.../inverted_index/analysis_factory_mgr.cpp | 7 +
.../inverted_index/analyzer/analyzer.cpp | 6 +-
.../analyzer/custom_analyzer_config.h | 1 -
.../char_filter/icu_normalizer_char_filter.cpp | 91 ++
.../char_filter/icu_normalizer_char_filter.h | 51 ++
.../icu_normalizer_char_filter_factory.h | 119 +++
.../normalizer/custom_normalizer.cpp | 110 +++
.../inverted_index/normalizer/custom_normalizer.h | 73 ++
.../normalizer/custom_normalizer_config.cpp | 49 ++
.../custom_normalizer_config.h} | 40 +-
.../token_filter/icu_normalizer_filter.cpp | 72 ++
.../token_filter/icu_normalizer_filter.h | 46 +-
.../token_filter/icu_normalizer_filter_factory.h | 105 +++
.../rowset/segment_v2/inverted_index_writer.cpp | 2 +-
be/src/olap/tablet_schema.h | 3 +-
be/src/pipeline/pipeline_fragment_context.cpp | 6 +-
be/src/runtime/index_policy/index_policy_mgr.cpp | 82 +-
be/src/runtime/index_policy/index_policy_mgr.h | 21 +-
.../aggregate_function_regr_union.h | 250 ++++--
be/src/vec/columns/column.cpp | 61 ++
be/src/vec/columns/column.h | 11 +
be/src/vec/core/block.cpp | 1 +
.../data_types/serde/data_type_nullable_serde.cpp | 10 +-
be/src/vec/exprs/vectorized_fn_call.cpp | 1 +
be/src/vec/exprs/vmatch_predicate.cpp | 2 +-
be/src/vec/functions/cast/cast_base.cpp | 18 +-
.../functions/cast/cast_to_basic_number_common.h | 26 +-
be/src/vec/functions/cast/cast_to_boolean.h | 26 +-
be/src/vec/functions/cast/cast_to_date.h | 67 +-
be/src/vec/functions/cast/cast_to_decimal.h | 80 +-
be/src/vec/functions/cast/cast_to_float.h | 2 +-
be/src/vec/functions/cast/cast_to_int.h | 37 +-
be/src/vec/functions/cast/cast_to_ip.h | 25 +-
be/src/vec/functions/cast/cast_to_jsonb.h | 9 +-
be/src/vec/functions/function_string.cpp | 1 +
be/src/vec/functions/function_string.h | 146 ++++
be/src/vec/functions/function_tokenize.cpp | 5 +-
be/src/vec/functions/match.cpp | 6 +-
be/test/olap/inverted_index_parser_test.cpp | 17 +
.../olap/rowset/segment_v2/column_reader_test.cpp | 1 -
.../inverted_index/ananlyzer/analyzer_test.cpp | 16 +-
.../icu_normalizer_char_filter_factory_test.cpp | 268 ++++++
.../icu_normalizer_filter_factory_test.cpp | 295 +++++++
.../segment_v2/inverted_index/util/reader_test.cpp | 2 +-
be/test/olap/tablet_schema_index_test.cpp | 6 +-
be/test/vec/columns/column_self_check.cpp | 123 +++
be/test/vec/function/cast/cast_to_integer.cpp | 7 +-
be/test/vec/function/function_match_test.cpp | 4 +-
be/test/vec/function/function_string_test.cpp | 91 ++
contrib/faiss | 2 +-
.../main/java/org/apache/doris/common/Config.java | 6 +
fe/fe-core/pom.xml | 5 +
.../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 2 +
.../antlr4/org/apache/doris/nereids/DorisParser.g4 | 8 +-
.../org/apache/doris/analysis/DescriptorTable.java | 5 +
.../apache/doris/analysis/InvertedIndexUtil.java | 86 +-
.../org/apache/doris/analysis/MatchPredicate.java | 8 +-
.../doris/catalog/BuiltinScalarFunctions.java | 2 +
.../main/java/org/apache/doris/catalog/Index.java | 14 +-
.../RecursiveCteTempTable.java} | 21 +-
.../java/org/apache/doris/catalog/TableIf.java | 19 +-
.../cache/NereidsSortedPartitionsCacheManager.java | 39 +-
.../org/apache/doris/datasource/ExternalTable.java | 14 +
.../doris/datasource/hive/HMSExternalTable.java | 15 +
.../doris/datasource/hive/HiveMetaStoreCache.java | 143 ++--
.../ICUNormalizerCharFilterValidator.java | 75 ++
.../ICUNormalizerTokenFilterValidator.java | 64 ++
.../org/apache/doris/indexpolicy/IndexPolicy.java | 7 +-
.../apache/doris/indexpolicy/IndexPolicyMgr.java | 159 +++-
.../doris/indexpolicy/IndexPolicyTypeEnum.java | 3 +-
.../insertoverwrite/InsertOverwriteManager.java | 6 +-
.../org/apache/doris/nereids/CascadesContext.java | 41 +-
.../glue/translator/PhysicalPlanTranslator.java | 105 ++-
.../doris/nereids/parser/LogicalPlanBuilder.java | 39 +-
.../apache/doris/nereids/pattern/MemoPatterns.java | 24 +
.../processor/post/RuntimeFilterPruner.java | 9 +
.../processor/pre/PullUpSubqueryAliasToCTE.java | 16 +-
.../properties/ChildOutputPropertyDeriver.java | 20 +
.../nereids/properties/RequestPropertyDeriver.java | 11 +
.../org/apache/doris/nereids/rules/RuleSet.java | 6 +
.../org/apache/doris/nereids/rules/RuleType.java | 3 +
.../doris/nereids/rules/analysis/AnalyzeCTE.java | 163 +++-
.../doris/nereids/rules/analysis/BindRelation.java | 32 +-
.../nereids/rules/analysis/CollectRelation.java | 11 +-
.../nereids/rules/analysis/SubExprAnalyzer.java | 4 +-
.../expression/rules/SortedPartitionRanges.java | 44 +
...eChildToPhysicalRecursiveCteRecursiveChild.java | 37 +
...RecursiveCteScanToPhysicalRecursiveCteScan.java | 42 +
.../LogicalRecursiveCteToPhysicalRecursiveCte.java | 39 +
.../nereids/rules/rewrite/AdjustNullable.java | 48 ++
.../doris/nereids/rules/rewrite/CTEInline.java | 44 +-
.../doris/nereids/rules/rewrite/ColumnPruning.java | 55 ++
.../rules/rewrite/PruneFileScanPartition.java | 12 +-
.../doris/nereids/stats/StatsCalculator.java | 109 +++
.../trees/copier/LogicalPlanDeepCopier.java | 26 +
.../doris/nereids/trees/expressions/Cast.java | 13 -
.../trees/expressions/functions/scalar/Date.java | 1 +
.../scalar/{Date.java => UnicodeNormalize.java} | 55 +-
.../expressions/visitor/ScalarFunctionVisitor.java | 5 +
.../apache/doris/nereids/trees/plans/PlanType.java | 9 +
.../trees/plans/algebra/RecursiveCte.java} | 29 +-
.../commands/CreateIndexNormalizerCommand.java | 67 ++
.../plans/commands/DropIndexNormalizerCommand.java | 63 ++
.../plans/commands/ShowIndexNormalizerCommand.java | 61 ++
.../plans/commands/UpdateMvByPartitionCommand.java | 3 +-
.../trees/plans/commands/info/IndexDefinition.java | 14 +-
.../worker/job/UnassignedJobBuilder.java | 13 +
.../worker/job/UnassignedRecursiveCteScanJob.java | 65 ++
.../nereids/trees/plans/logical/LogicalCTE.java | 27 +-
.../trees/plans/logical/LogicalRecursiveCte.java | 284 ++++++
.../logical/LogicalRecursiveCteRecursiveChild.java | 117 +++
.../plans/logical/LogicalRecursiveCteScan.java | 87 ++
.../trees/plans/logical/LogicalSubQueryAlias.java | 22 +
.../trees/plans/physical/PhysicalRecursiveCte.java | 292 +++++++
.../PhysicalRecursiveCteRecursiveChild.java | 121 +++
.../plans/physical/PhysicalRecursiveCteScan.java | 85 ++
.../trees/plans/visitor/CommandVisitor.java | 18 +
.../nereids/trees/plans/visitor/PlanVisitor.java | 22 +
.../trees/plans/visitor/RelationVisitor.java | 10 +
.../org/apache/doris/planner/RecursiveCteNode.java | 86 ++
.../apache/doris/planner/RecursiveCteScanNode.java | 121 +++
.../java/org/apache/doris/qe/SessionVariable.java | 9 +-
.../doris/qe/runtime/ThriftPlansBuilder.java | 151 +++-
.../nereids/postprocess/RuntimeFilterTest.java | 29 +
.../nereids/rules/analysis/AnalyzeCTETest.java | 219 +++++
.../doris/nereids/rules/rewrite/CTEInlineTest.java | 84 ++
.../nereids/rules/rewrite/ColumnPruningTest.java | 30 +
gensrc/thrift/AgentService.thrift | 3 +-
gensrc/thrift/Exprs.thrift | 2 +-
.../test_remote_doris_agg_table_select.out | 54 ++
.../test_remote_doris_unique_table_select.out | 32 +
.../test_remote_doris_variant_select.out | 81 ++
.../analyzer/test_custom_normalizer.out | 15 +
.../analyzer/test_unicode_normalize.out | 19 +
.../limit_push_down/limit_push_down.out | 32 +-
.../support_type/regr_intercept/regr_intercept.out | 8 +-
.../support_type/regr_slope/regr_slope.out | 8 +-
.../data/rec_cte_p0/rec_cte/rec_cte.out | 953 +++++++++++++++++++++
.../rec_cte_from_ck_doc/rec_cte_from_ck_doc.out | 29 +
.../rec_cte_from_duckdb_doc.out | 30 +
.../rec_cte_from_mysql_doc.out | 42 +
.../test_non_overlap_seg_heavy_sc.out | 2 +-
.../test_remote_doris_agg_table_select.groovy | 307 +++++++
.../test_remote_doris_unique_table_select.groovy | 238 +++++
.../test_remote_doris_variant_select.groovy | 148 ++++
.../cast/test_cast_to_complex_types_strict.groovy | 6 +
.../analyzer/test_custom_normalizer.groovy | 131 +++
.../analyzer/test_unicode_normalize.groovy | 79 ++
.../suites/prepared_stmt_p0/prepared_stmt.groovy | 31 +-
.../query_p0/aggregate/test_regr_intercept.groovy | 18 +-
.../query_p0/aggregate/test_regr_slope.groovy | 18 +-
.../suites/rec_cte_p0/rec_cte/rec_cte.groovy | 271 ++++++
.../rec_cte_from_ck_doc/rec_cte_from_ck_doc.groovy | 144 ++++
.../rec_cte_from_duckdb_doc.groovy | 185 ++++
.../rec_cte_from_mysql_doc.groovy | 140 +++
159 files changed, 8665 insertions(+), 756 deletions(-)
create mode 100644
be/src/olap/rowset/segment_v2/inverted_index/char_filter/icu_normalizer_char_filter.cpp
create mode 100644
be/src/olap/rowset/segment_v2/inverted_index/char_filter/icu_normalizer_char_filter.h
create mode 100644
be/src/olap/rowset/segment_v2/inverted_index/char_filter/icu_normalizer_char_filter_factory.h
create mode 100644
be/src/olap/rowset/segment_v2/inverted_index/normalizer/custom_normalizer.cpp
create mode 100644
be/src/olap/rowset/segment_v2/inverted_index/normalizer/custom_normalizer.h
create mode 100644
be/src/olap/rowset/segment_v2/inverted_index/normalizer/custom_normalizer_config.cpp
copy
be/src/olap/rowset/segment_v2/inverted_index/{analyzer/custom_analyzer_config.h
=> normalizer/custom_normalizer_config.h} (59%)
create mode 100644
be/src/olap/rowset/segment_v2/inverted_index/token_filter/icu_normalizer_filter.cpp
copy
fe/fe-core/src/main/java/org/apache/doris/indexpolicy/IndexPolicyTypeEnum.java
=>
be/src/olap/rowset/segment_v2/inverted_index/token_filter/icu_normalizer_filter.h
(55%)
create mode 100644
be/src/olap/rowset/segment_v2/inverted_index/token_filter/icu_normalizer_filter_factory.h
create mode 100644
be/test/olap/rowset/segment_v2/inverted_index/char_filter/icu_normalizer_char_filter_factory_test.cpp
create mode 100644
be/test/olap/rowset/segment_v2/inverted_index/token_filter/icu_normalizer_filter_factory_test.cpp
create mode 100644 be/test/vec/columns/column_self_check.cpp
copy
fe/fe-core/src/main/java/org/apache/doris/{indexpolicy/IndexPolicyTypeEnum.java
=> catalog/RecursiveCteTempTable.java} (56%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/indexpolicy/ICUNormalizerCharFilterValidator.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/indexpolicy/ICUNormalizerTokenFilterValidator.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteRecursiveChildToPhysicalRecursiveCteRecursiveChild.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteScanToPhysicalRecursiveCteScan.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalRecursiveCteToPhysicalRecursiveCte.java
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/{Date.java
=> UnicodeNormalize.java} (54%)
copy
fe/fe-core/src/main/java/org/apache/doris/{indexpolicy/IndexPolicyTypeEnum.java
=> nereids/trees/plans/algebra/RecursiveCte.java} (56%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/CreateIndexNormalizerCommand.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/DropIndexNormalizerCommand.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowIndexNormalizerCommand.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/distribute/worker/job/UnassignedRecursiveCteScanJob.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCte.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteRecursiveChild.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalRecursiveCteScan.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCte.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteRecursiveChild.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalRecursiveCteScan.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteNode.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/planner/RecursiveCteScanNode.java
create mode 100644
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/CTEInlineTest.java
create mode 100644
regression-test/data/external_table_p0/remote_doris/test_remote_doris_agg_table_select.out
create mode 100644
regression-test/data/external_table_p0/remote_doris/test_remote_doris_unique_table_select.out
create mode 100644
regression-test/data/external_table_p0/remote_doris/test_remote_doris_variant_select.out
create mode 100644
regression-test/data/inverted_index_p0/analyzer/test_custom_normalizer.out
create mode 100644
regression-test/data/inverted_index_p0/analyzer/test_unicode_normalize.out
create mode 100644 regression-test/data/rec_cte_p0/rec_cte/rec_cte.out
create mode 100644
regression-test/data/rec_cte_p0/rec_cte_from_ck_doc/rec_cte_from_ck_doc.out
create mode 100644
regression-test/data/rec_cte_p0/rec_cte_from_duckdb_doc/rec_cte_from_duckdb_doc.out
create mode 100644
regression-test/data/rec_cte_p0/rec_cte_from_mysql_doc/rec_cte_from_mysql_doc.out
create mode 100644
regression-test/suites/external_table_p0/remote_doris/test_remote_doris_agg_table_select.groovy
create mode 100644
regression-test/suites/external_table_p0/remote_doris/test_remote_doris_unique_table_select.groovy
create mode 100644
regression-test/suites/external_table_p0/remote_doris/test_remote_doris_variant_select.groovy
create mode 100644
regression-test/suites/inverted_index_p0/analyzer/test_custom_normalizer.groovy
create mode 100644
regression-test/suites/inverted_index_p0/analyzer/test_unicode_normalize.groovy
create mode 100644 regression-test/suites/rec_cte_p0/rec_cte/rec_cte.groovy
create mode 100644
regression-test/suites/rec_cte_p0/rec_cte_from_ck_doc/rec_cte_from_ck_doc.groovy
create mode 100644
regression-test/suites/rec_cte_p0/rec_cte_from_duckdb_doc/rec_cte_from_duckdb_doc.groovy
create mode 100644
regression-test/suites/rec_cte_p0/rec_cte_from_mysql_doc/rec_cte_from_mysql_doc.groovy
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
