This is an automated email from the ASF dual-hosted git repository.
yangzhg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6a05879 [Feature][Step1] Support lateral view FE part (#6745)
6a05879 is described below
commit 6a058792af8d9120eb85633844fa6f178bbb7a05
Author: EmmyMiao87 <[email protected]>
AuthorDate: Wed Oct 13 11:37:12 2021 +0800
[Feature][Step1] Support lateral view FE part (#6745)
* [Feature] Support lateral view
The syntax:
```
select k1, e1 from test lateral view explode_split(k1, ",") tmp as e1;
```
```explode_split``` is a special function of doris,
which is used to separate the string column according to the specified
split string,
and then convert the row to column.
This is a conforming function of string separation + table function,
and its behavior is equivalent to explode in hive ```explode(split(string,
string))```
The implement:
A tablefunction operator is added to the implementation to handle the
syntax of the lateral view separately.
The query plan is following:
```
MySQL [test]> explain select k1, e1 from test_explode lateral view
explode_split (k2, ",") tmp as e1;
+---------------------------------------------------------------------------+
| Explain String
|
+---------------------------------------------------------------------------+
| PLAN FRAGMENT 0
|
| OUTPUT EXPRS:`k1` | `e1`
|
|
|
| RESULT SINK
|
|
|
| 1:TABLE FUNCTION NODE
|
| | table function: explode_split(`k2`, ',')
|
| |
|
| 0:OlapScanNode
|
| TABLE: test_explode
|
+---------------------------------------------------------------------------+
```
* Add ut
* Add multi table function node
* Add session variables 'enable_lateral_view'
* Fix ut
---
fe/fe-core/src/main/cup/sql_parser.cup | 43 ++++-
.../org/apache/doris/analysis/BaseTableRef.java | 15 +-
.../main/java/org/apache/doris/analysis/Expr.java | 9 +
.../apache/doris/analysis/FunctionCallExpr.java | 20 ++-
.../org/apache/doris/analysis/InlineViewRef.java | 5 +-
.../org/apache/doris/analysis/LateralViewRef.java | 127 +++++++++++++
.../java/org/apache/doris/analysis/SelectStmt.java | 9 +
.../java/org/apache/doris/analysis/TableRef.java | 16 +-
.../java/org/apache/doris/catalog/Catalog.java | 4 +
.../java/org/apache/doris/catalog/FunctionSet.java | 40 ++++-
.../apache/doris/planner/DistributedPlanner.java | 13 +-
.../apache/doris/planner/SingleNodePlanner.java | 22 ++-
.../apache/doris/planner/TableFunctionNode.java | 81 +++++++++
.../java/org/apache/doris/qe/SessionVariable.java | 13 ++
fe/fe-core/src/main/jflex/sql_scanner.flex | 1 +
.../doris/planner/RuntimeFilterGeneratorTest.java | 3 +-
.../doris/planner/TableFunctionPlanTest.java | 199 +++++++++++++++++++++
.../org/apache/doris/utframe/UtFrameUtils.java | 6 +-
gensrc/thrift/PlanNodes.thrift | 7 +
19 files changed, 612 insertions(+), 21 deletions(-)
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup
b/fe/fe-core/src/main/cup/sql_parser.cup
index 20e9c8e..be00fd2 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -236,7 +236,7 @@ parser code {:
terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE, KW_ALIAS, KW_ALL,
KW_ALTER, KW_AND, KW_ANTI, KW_APPEND, KW_AS, KW_ASC, KW_AUTHORS, KW_ARRAY,
KW_BACKEND, KW_BACKUP, KW_BETWEEN, KW_BEGIN, KW_BIGINT, KW_BINLOG,
KW_BITMAP, KW_BITMAP_UNION, KW_BLOB, KW_BOOLEAN, KW_BROKER, KW_BACKENDS, KW_BY,
KW_BUILTIN,
KW_CANCEL, KW_CASE, KW_CAST, KW_CHAIN, KW_CHAR, KW_CHARSET, KW_CHECK,
KW_CLUSTER, KW_CLUSTERS, KW_CLEAN,
- KW_COLLATE, KW_COLLATION, KW_COLUMN, KW_COLON, KW_COLUMNS, KW_COMMENT,
KW_COMMIT, KW_COMMITTED,
+ KW_COLLATE, KW_COLLATION, KW_COLUMN, KW_COLUMNS, KW_COMMENT, KW_COMMIT,
KW_COMMITTED,
KW_CONFIG, KW_CONNECTION, KW_CONNECTION_ID, KW_CONSISTENT, KW_CONVERT,
KW_COUNT, KW_CREATE, KW_CROSS, KW_CUBE, KW_CURRENT, KW_CURRENT_USER,
KW_DATA, KW_DATABASE, KW_DATABASES, KW_DATE, KW_DATETIME, KW_DAY,
KW_DECIMAL, KW_DECOMMISSION, KW_DEFAULT, KW_DESC, KW_DESCRIBE,
KW_DELETE, KW_UPDATE, KW_DISTINCT, KW_DISTINCTPC, KW_DISTINCTPCSA,
KW_DISTRIBUTED, KW_DISTRIBUTION, KW_DYNAMIC, KW_BUCKETS, KW_DIV, KW_DOUBLE,
KW_DROP, KW_DROPP, KW_DUPLICATE,
@@ -250,9 +250,9 @@ terminal String KW_ADD, KW_ADMIN, KW_AFTER, KW_AGGREGATE,
KW_ALIAS, KW_ALL, KW_A
KW_JOB, KW_JOIN,
KW_KEY, KW_KEYS, KW_KILL,
KW_LABEL, KW_LARGEINT, KW_LAST, KW_LEFT, KW_LESS, KW_LEVEL, KW_LIKE,
KW_LIMIT, KW_LINK, KW_LIST, KW_LOAD,
- KW_LOCAL, KW_LOCATION,
+ KW_LOCAL, KW_LOCATION, KW_LATERAL,
KW_MAP, KW_MATERIALIZED, KW_MAX, KW_MAX_VALUE, KW_MERGE, KW_MIN,
KW_MINUTE, KW_MINUS, KW_MIGRATE, KW_MIGRATIONS, KW_MODIFY, KW_MONTH,
- KW_NAME, KW_NAMED_STRUCT, KW_NAMES, KW_NEGATIVE, KW_NO, KW_NOT, KW_NULL,
KW_NULLS,
+ KW_NAME, KW_NAMES, KW_NEGATIVE, KW_NO, KW_NOT, KW_NULL, KW_NULLS,
KW_OBSERVER, KW_OFFSET, KW_ON, KW_ONLY, KW_OPEN, KW_OR, KW_ORDER,
KW_OUTER, KW_OUTFILE, KW_OVER,
KW_PARAMETER, KW_PARTITION, KW_PARTITIONS, KW_PASSWORD,
KW_LDAP_ADMIN_PASSWORD, KW_PATH, KW_PAUSE, KW_PIPE, KW_PRECEDING,
KW_PLUGIN, KW_PLUGINS,
@@ -385,9 +385,11 @@ nonterminal AnalyticWindow.Boundary window_boundary;
nonterminal SlotRef column_ref;
nonterminal FunctionCallExpr column_subscript;
nonterminal ArrayList<TableRef> table_ref_list, base_table_ref_list;
+nonterminal ArrayList<LateralViewRef> opt_lateral_view_ref_list,
lateral_view_ref_list;
nonterminal FromClause from_clause;
nonterminal TableRef table_ref;
nonterminal TableRef base_table_ref;
+nonterminal LateralViewRef lateral_view_ref;
nonterminal WithClause opt_with_clause;
nonterminal ArrayList<View> with_view_def_list;
nonterminal View with_view_def;
@@ -3998,8 +4000,9 @@ base_table_ref_list ::=
base_table_ref ::=
table_name:name opt_partition_names:partitionNames opt_table_alias:alias
opt_common_hints:commonHints
+ opt_lateral_view_ref_list:lateralViewRefList
{:
- RESULT = new TableRef(name, alias, partitionNames, commonHints);
+ RESULT = new TableRef(name, alias, partitionNames, commonHints,
lateralViewRefList);
:}
;
@@ -4075,6 +4078,38 @@ partition_names ::=
:}
;
+opt_lateral_view_ref_list ::=
+ /* empty */
+ {:
+ RESULT = null;
+ :}
+ | lateral_view_ref_list:lateralViewRefList
+ {:
+ RESULT = lateralViewRefList;
+ :}
+ ;
+
+lateral_view_ref_list ::=
+ lateral_view_ref:lateralViewRef
+ {:
+ ArrayList<LateralViewRef> list = new ArrayList<LateralViewRef>();
+ list.add(lateralViewRef);
+ RESULT = list;
+ :}
+ | lateral_view_ref:lateralViewRef lateral_view_ref_list:lateralViewRefList
+ {:
+ lateralViewRefList.add(lateralViewRef);
+ RESULT = lateralViewRefList;
+ :}
+ ;
+
+lateral_view_ref ::=
+ KW_LATERAL KW_VIEW function_call_expr:fnExpr ident:viewName KW_AS
ident:columnName
+ {:
+ RESULT = new LateralViewRef(fnExpr, viewName, columnName);
+ :}
+ ;
+
join_operator ::=
opt_inner KW_JOIN
{: RESULT = JoinOperator.INNER_JOIN; :}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/BaseTableRef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/BaseTableRef.java
index f46e60a..f39db3a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BaseTableRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BaseTableRef.java
@@ -18,7 +18,7 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Table;
-import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.UserException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -63,7 +63,7 @@ public class BaseTableRef extends TableRef {
* Register this table ref and then analyze the Join clause.
*/
@Override
- public void analyze(Analyzer analyzer) throws AnalysisException {
+ public void analyze(Analyzer analyzer) throws UserException {
name = analyzer.getFqTableName(name);
name.analyze(analyzer);
desc = analyzer.registerTableRef(this);
@@ -71,6 +71,17 @@ public class BaseTableRef extends TableRef {
analyzeJoin(analyzer);
analyzeSortHints();
analyzeHints();
+ analyzeLateralViewRef(analyzer);
+ }
+
+ private void analyzeLateralViewRef(Analyzer analyzer) throws UserException
{
+ if (lateralViewRefs == null) {
+ return;
+ }
+ for (LateralViewRef lateralViewRef : lateralViewRefs) {
+ lateralViewRef.setRelatedTable(table);
+ lateralViewRef.analyze(analyzer);
+ }
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
index a7098a9..d27000c 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java
@@ -1575,6 +1575,15 @@ abstract public class Expr extends TreeNode<Expr>
implements ParseNode, Cloneabl
return f;
}
+ protected Function getTableFunction(String name, Type[] argTypes,
+ Function.CompareMode mode) {
+ FunctionName fnName = new FunctionName(name);
+ Function searchDesc = new Function(fnName, Arrays.asList(argTypes),
Type.INVALID, false,
+ VectorizedUtil.isVectorized());
+ Function f = Catalog.getCurrentCatalog().getTableFunction(searchDesc,
mode);
+ return f;
+ }
+
/**
* Pushes negation to the individual operands of a predicate
* tree rooted at 'root'.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index c93efd8..d10bc32 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -18,8 +18,8 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.AggregateFunction;
-import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.AliasFunction;
+import org.apache.doris.catalog.ArrayType;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Function;
@@ -52,9 +52,9 @@ import org.apache.logging.log4j.Logger;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.text.StringCharacterIterator;
import java.util.Arrays;
import java.util.List;
-import java.text.StringCharacterIterator;
// TODO: for aggregations, we need to unify the code paths for builtins and
UDAs.
public class FunctionCallExpr extends Expr {
private static final Logger LOG =
LogManager.getLogger(FunctionCallExpr.class);
@@ -64,6 +64,8 @@ public class FunctionCallExpr extends Expr {
// check analytic function
private boolean isAnalyticFnCall = false;
+ // check table function
+ private boolean isTableFnCall = false;
// Indicates whether this is a merge aggregation function that should use
the merge
// instead of the update symbol. This flag also affects the behavior of
@@ -87,6 +89,10 @@ public class FunctionCallExpr extends Expr {
isAnalyticFnCall = v;
}
+ public void setTableFnCall(boolean tableFnCall) {
+ isTableFnCall = tableFnCall;
+ }
+
public Function getFn() {
return fn;
}
@@ -675,6 +681,16 @@ public class FunctionCallExpr extends Expr {
fn = getBuiltinFunction(analyzer, fnName.getFunction(), new
Type[]{compatibleType},
Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
} else {
+ // now first find table function in table function sets
+ if (isTableFnCall) {
+ Type[] childTypes = collectChildReturnTypes();
+ fn = getTableFunction(fnName.getFunction(), childTypes,
+ Function.CompareMode.IS_NONSTRICT_SUPERTYPE_OF);
+ if (fn == null) {
+ throw new AnalysisException("Doris only support
`explode_split(varchar, varchar)` table function");
+ }
+ return;
+ }
// now first find function in built-in functions
if (Strings.isNullOrEmpty(fnName.getDb())) {
Type[] childTypes = collectChildReturnTypes();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
index 807949e..cd8b045 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/InlineViewRef.java
@@ -266,8 +266,9 @@ public class InlineViewRef extends TableRef {
}
columnSet.add(colAlias);
- columnList.add(new Column(colAlias,
selectItemExpr.getType().getPrimitiveType(),
- selectItemExpr.isNullable()));
+ columnList.add(new Column(colAlias, selectItemExpr.getType(),
+ false, null, selectItemExpr.isNullable(),
+ null, ""));
}
InlineView inlineView = (view != null) ? new InlineView(view,
columnList) : new InlineView(getExplicitAlias(), columnList);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java
new file mode 100644
index 0000000..e6576ec
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/LateralViewRef.java
@@ -0,0 +1,127 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.analysis;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.FunctionSet;
+import org.apache.doris.catalog.InlineView;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.Table;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.UserException;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+import java.util.List;
+
+/**
+ * lateralView: LATERAL VIEW udtf(expression) tableAlias AS columnAlias (','
columnAlias)
+ * fromClause: FROM baseTable (lateralView)
+ */
+public class LateralViewRef extends TableRef {
+
+ private Expr expr;
+ private String viewName;
+ private String columnName;
+ private Table relatedTable;
+
+ // after analyzed
+ private FunctionCallExpr fnExpr;
+ private Column originColumn;
+ private SlotRef originSlotRef;
+ private InlineView view;
+ private SlotRef explodeSlotRef;
+
+ public LateralViewRef(Expr expr, String viewName, String columnName) {
+ super(null, viewName);
+ this.expr = expr;
+ this.viewName = viewName;
+ this.columnName = columnName;
+ }
+
+ public void setRelatedTable(Table relatedTable) {
+ this.relatedTable = relatedTable;
+ }
+
+ public FunctionCallExpr getFnExpr() {
+ return fnExpr;
+ }
+
+ @Override
+ public void analyze(Analyzer analyzer) throws UserException {
+ if (!analyzer.getContext().getSessionVariable().isEnableLateralView())
{
+ throw new AnalysisException("The session variables
`enable_lateral_view` is false");
+ }
+
+ if (isAnalyzed) {
+ return;
+ }
+ Preconditions.checkNotNull(relatedTable);
+ // analyze table
+ if (!(relatedTable instanceof OlapTable)) {
+ throw new AnalysisException("Only doris table could be exploded");
+ }
+ // analyze function and slot
+ if (!(expr instanceof FunctionCallExpr)) {
+ throw new AnalysisException("Only support function call expr in
lateral view");
+ }
+ fnExpr = (FunctionCallExpr) expr;
+ fnExpr.setTableFnCall(true);
+ fnExpr.analyze(analyzer);
+ if
(!fnExpr.getFnName().getFunction().equals(FunctionSet.EXPLODE_SPLIT)) {
+ throw new AnalysisException("Only support explode function in
lateral view");
+ }
+ if (!(fnExpr.getChild(0) instanceof SlotRef)) {
+ throw new AnalysisException("Explode column must be varchar
column");
+ }
+ if (!(fnExpr.getChild(1) instanceof StringLiteral)) {
+ throw new AnalysisException("Split separator of explode must be a
string const");
+ }
+ originSlotRef = ((SlotRef) fnExpr.getChild(0));
+ originColumn = originSlotRef.getColumn();
+ if (originColumn == null) {
+ throw new AnalysisException("The explode column must be a real
column in table");
+ }
+ // analyze lateral view
+ desc = analyzer.registerTableRef(this);
+ explodeSlotRef = new SlotRef(new TableName(null, viewName),
columnName);
+ explodeSlotRef.analyze(analyzer);
+ isAnalyzed = true; // true now that we have assigned desc
+ }
+
+ @Override
+ public TupleDescriptor createTupleDescriptor(Analyzer analyzer) throws
AnalysisException {
+ // Create a fake catalog table for the lateral view
+ List<Column> columnList = Lists.newArrayList();
+ columnList.add(new Column(columnName, originColumn.getType(),
+ false, null, originColumn.isAllowNull(),
+ null, ""));
+ view = new InlineView(viewName, columnList);
+
+ // Create the non-materialized tuple and set the fake table in it.
+ TupleDescriptor result = analyzer.getDescTbl().createTupleDescriptor();
+ result.setTable(view);
+ return result;
+ }
+
+ public void materializeRequiredSlots() {
+ originSlotRef.getDesc().setIsMaterialized(true);
+ explodeSlotRef.getDesc().setIsMaterialized(true);
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 0f2167e..1d19f79 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -692,6 +692,15 @@ public class SelectStmt extends QueryStmt {
materializeSlots(analyzer, havingConjuncts);
aggInfo.materializeRequiredSlots(analyzer, baseTblSmap);
}
+
+ // materialized all lateral view column and origin column
+ for (TableRef tableRef : fromClause_.getTableRefs()) {
+ if (tableRef.lateralViewRefs != null) {
+ for (LateralViewRef lateralViewRef : tableRef.lateralViewRefs)
{
+ lateralViewRef.materializeRequiredSlots();
+ }
+ }
+ }
}
protected void reorderTable(Analyzer analyzer) throws AnalysisException {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
index f18d574..256d0f3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java
@@ -17,7 +17,6 @@
package org.apache.doris.analysis;
-import org.apache.commons.collections.CollectionUtils;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
@@ -34,6 +33,7 @@ import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
+import org.apache.commons.collections.CollectionUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -94,6 +94,7 @@ public class TableRef implements ParseNode, Writable {
private ArrayList<String> joinHints;
private ArrayList<String> sortHints;
private ArrayList<String> commonHints; //The Hints is set by user
+ protected ArrayList<LateralViewRef> lateralViewRefs;
private boolean isForcePreAggOpened;
// ///////////////////////////////////////
// BEGIN: Members that need to be reset()
@@ -141,19 +142,21 @@ public class TableRef implements ParseNode, Writable {
}
public TableRef(TableName name, String alias, PartitionNames
partitionNames) {
- this(name, alias, partitionNames, null);
+ this(name, alias, partitionNames, null, null);
}
- public TableRef(TableName name, String alias, PartitionNames
partitionNames, ArrayList<String> commonHints) {
+ public TableRef(TableName name, String alias, PartitionNames
partitionNames, ArrayList<String> commonHints,
+ ArrayList<LateralViewRef> lateralViewRefs) {
this.name = name;
if (alias != null) {
- aliases_ = new String[] { alias };
+ aliases_ = new String[]{alias};
hasExplicitAlias_ = true;
} else {
hasExplicitAlias_ = false;
}
this.partitionNames = partitionNames;
this.commonHints = commonHints;
+ this.lateralViewRefs = lateralViewRefs;
isAnalyzed = false;
}
// Only used to clone
@@ -182,6 +185,7 @@ public class TableRef implements ParseNode, Writable {
allMaterializedTupleIds_ =
Lists.newArrayList(other.allMaterializedTupleIds_);
correlatedTupleIds_ = Lists.newArrayList(other.correlatedTupleIds_);
desc = other.desc;
+ lateralViewRefs = other.lateralViewRefs;
}
public PartitionNames getPartitionNames() {
@@ -329,6 +333,10 @@ public class TableRef implements ParseNode, Writable {
return sortColumn;
}
+ public ArrayList<LateralViewRef> getLateralViewRefs() {
+ return lateralViewRefs;
+ }
+
protected void analyzeSortHints() throws AnalysisException {
if (sortHints == null) {
return;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index f846b01..e806fea 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -5888,6 +5888,10 @@ public class Catalog {
return functionSet.getBulitinFunctions();
}
+ public Function getTableFunction(Function desc, Function.CompareMode mode)
{
+ return functionSet.getFunction(desc, mode, true);
+ }
+
public boolean isNullResultWithOneNullParamFunction(String funcName) {
return functionSet.isNullResultWithOneNullParamFunctions(funcName);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
index d318193..379c567 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
@@ -17,7 +17,6 @@
package org.apache.doris.catalog;
-import com.google.common.base.Preconditions;
import org.apache.doris.analysis.ArithmeticExpr;
import org.apache.doris.analysis.BinaryPredicate;
import org.apache.doris.analysis.CastExpr;
@@ -27,6 +26,7 @@ import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.LikePredicate;
import org.apache.doris.builtins.ScalarBuiltins;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
@@ -51,6 +51,7 @@ public class
FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
// FunctionResolutionOrder.
private final HashMap<String, List<Function>> functions;
private final HashMap<String, List<Function>> vectorizedFunctions;
+ private final HashMap<String, List<Function>> tableFunctions;
// For most build-in functions, it will return NullLiteral when params
contain NullLiteral.
// But a few functions need to handle NullLiteral differently, such as
"if". It need to add
// an attribute to LiteralExpr to mark null and check the attribute to
decide whether to
@@ -66,6 +67,7 @@ public class
FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
public FunctionSet() {
functions = Maps.newHashMap();
vectorizedFunctions = Maps.newHashMap();
+ tableFunctions = Maps.newHashMap();
}
public void init() {
@@ -82,6 +84,9 @@ public class
FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
LikePredicate.initBuiltins(this);
InPredicate.initBuiltins(this);
AliasFunction.initBuiltins(this);
+
+ // init table function
+ initTableFunction();
}
public void buildNullResultWithOneNullParamFunction(Set<String> funcNames)
{
@@ -977,7 +982,18 @@ public class
FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
.build();
public Function getFunction(Function desc, Function.CompareMode mode) {
- List<Function> fns = desc.isVectorized() ?
vectorizedFunctions.get(desc.functionName()) :
functions.get(desc.functionName());
+ return getFunction(desc, mode, false);
+ }
+
+ public Function getFunction(Function desc, Function.CompareMode mode,
boolean isTableFunction) {
+ List<Function> fns;
+ if (isTableFunction) {
+ fns = tableFunctions.get(desc.functionName());
+ } else if (desc.isVectorized()) {
+ fns = vectorizedFunctions.get(desc.functionName());
+ } else {
+ fns = functions.get(desc.functionName());
+ }
if (fns == null) {
return null;
}
@@ -1032,6 +1048,10 @@ public class
FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
final String functionName = desc.getFunctionName().getFunction();
final Type[] descArgTypes = desc.getArgs();
final Type[] candicateArgTypes = candicate.getArgs();
+ if (!(descArgTypes[0] instanceof ScalarType)
+ || !(candicateArgTypes[0] instanceof ScalarType)) {
+ return false;
+ }
if (functionName.equalsIgnoreCase("hex")
|| functionName.equalsIgnoreCase("greast")
|| functionName.equalsIgnoreCase("least")
@@ -1989,4 +2009,20 @@ public class
FunctionSet<min_initIN9doris_udf12DecimalV2ValEEEvPNS2_15FunctionCo
}
return builtinFunctions;
}
+
+
+ public static final String EXPLODE_SPLIT = "explode_split";
+
+ private void initTableFunction() {
+ // init explode_split function
+ ArrayList<Type> argsType = Lists.newArrayList();
+ argsType.add(Type.VARCHAR);
+ argsType.add(Type.VARCHAR);
+ Function explodeSplit = ScalarFunction.createBuiltin(
+ EXPLODE_SPLIT, Type.VARCHAR,
Function.NullableMode.DEPEND_ON_ARGUMENT, argsType, false,
+ "", "", "", true);
+ List<Function> explodeSplits = Lists.newArrayList();
+ explodeSplits.add(explodeSplit);
+ tableFunctions.put(EXPLODE_SPLIT, explodeSplits);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
index ddf578c..a77a3ce 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DistributedPlanner.java
@@ -200,6 +200,8 @@ public class DistributedPlanner {
if (root instanceof ScanNode) {
result = createScanFragment(root);
fragments.add(result);
+ } else if (root instanceof TableFunctionNode) {
+ result = createTableFunctionFragment(root, childFragments.get(0));
} else if (root instanceof HashJoinNode) {
Preconditions.checkState(childFragments.size() == 2);
result = createHashJoinFragment((HashJoinNode) root,
childFragments.get(1),
@@ -209,7 +211,7 @@ public class DistributedPlanner {
childFragments.get(0));
} else if (root instanceof SelectNode) {
result = createSelectNodeFragment((SelectNode) root,
childFragments);
- } else if (root instanceof SetOperationNode) {
+ } else if (root instanceof SetOperationNode) {
result = createSetOperationNodeFragment((SetOperationNode) root,
childFragments, fragments);
} else if (root instanceof MergeNode) {
result = createMergeNodeFragment((MergeNode) root, childFragments,
fragments);
@@ -288,12 +290,19 @@ public class DistributedPlanner {
}
}
+ private PlanFragment createTableFunctionFragment(PlanNode node,
PlanFragment childFragment) {
+ Preconditions.checkState(node instanceof TableFunctionNode);
+ node.setChild(0, childFragment.getPlanRoot());
+ childFragment.addPlanRoot(node);
+ return childFragment;
+ }
+
/**
* When broadcastCost and partitionCost are equal, there is no uniform
standard for which join implementation is better.
* Some scenarios are suitable for broadcast join, and some scenarios are
suitable for shuffle join.
* Therefore, we add a SessionVariable to help users choose a better join
implementation.
*/
- private boolean isBroadcastCostSmaller(long broadcastCost, long
partitionCost) {
+ private boolean isBroadcastCostSmaller(long broadcastCost, long
partitionCost) {
String joinMethod =
ConnectContext.get().getSessionVariable().getPreferJoinMethod();
if (joinMethod.equalsIgnoreCase("broadcast")) {
return broadcastCost <= partitionCost;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
index f10e676..3d0533b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java
@@ -35,6 +35,7 @@ import org.apache.doris.analysis.InPredicate;
import org.apache.doris.analysis.InlineViewRef;
import org.apache.doris.analysis.IsNullPredicate;
import org.apache.doris.analysis.JoinOperator;
+import org.apache.doris.analysis.LateralViewRef;
import org.apache.doris.analysis.LiteralExpr;
import org.apache.doris.analysis.NullLiteral;
import org.apache.doris.analysis.QueryStmt;
@@ -1859,7 +1860,12 @@ public class SingleNodePlanner {
private PlanNode createTableRefNode(Analyzer analyzer, TableRef tblRef,
SelectStmt selectStmt)
throws UserException, AnalysisException {
if (tblRef instanceof BaseTableRef) {
- return createScanNode(analyzer, tblRef, selectStmt);
+ PlanNode scanNode = createScanNode(analyzer, tblRef, selectStmt);
+ List<LateralViewRef> lateralViewRefs = tblRef.getLateralViewRefs();
+ if (lateralViewRefs != null && lateralViewRefs.size() != 0) {
+ return createTableFunctionNode(analyzer, scanNode,
lateralViewRefs);
+ }
+ return scanNode;
}
if (tblRef instanceof InlineViewRef) {
return createInlineViewPlan(analyzer, (InlineViewRef) tblRef);
@@ -1867,6 +1873,20 @@ public class SingleNodePlanner {
throw new UserException("unknown TableRef node");
}
+ private PlanNode createTableFunctionNode(Analyzer analyzer, PlanNode
inputNode,
+ List<LateralViewRef>
lateralViewRefs)
+ throws UserException {
+ Preconditions.checkNotNull(lateralViewRefs);
+ Preconditions.checkState(lateralViewRefs.size() > 0);
+ for (LateralViewRef lateralViewRef: lateralViewRefs) {
+ TableFunctionNode tableFunctionNode = new
TableFunctionNode(ctx_.getNextNodeId(), inputNode,
+ lateralViewRef);
+ tableFunctionNode.init(analyzer);
+ inputNode = tableFunctionNode;
+ }
+ return inputNode;
+ }
+
/**
* Create a plan tree corresponding to 'setOperands' for the given
SetOperationStmt.
* The individual operands' plan trees are attached to a single
SetOperationNode.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
new file mode 100644
index 0000000..4601f8a
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TableFunctionNode.java
@@ -0,0 +1,81 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner;
+
+import org.apache.doris.analysis.Analyzer;
+import org.apache.doris.analysis.FunctionCallExpr;
+import org.apache.doris.analysis.LateralViewRef;
+import org.apache.doris.common.UserException;
+import org.apache.doris.thrift.TExplainLevel;
+import org.apache.doris.thrift.TPlanNode;
+import org.apache.doris.thrift.TPlanNodeType;
+import org.apache.doris.thrift.TTableFunctionNode;
+
+public class TableFunctionNode extends PlanNode {
+
+ private LateralViewRef lateralViewRef;
+
+ private FunctionCallExpr fnCallExpr;
+
+ protected TableFunctionNode(PlanNodeId id, PlanNode inputNode,
LateralViewRef lateralViewRef) {
+ super(id, "TABLE FUNCTION NODE");
+ tupleIds.addAll(inputNode.getTupleIds());
+ tblRefIds.addAll(inputNode.getTupleIds());
+ tupleIds.add(lateralViewRef.getDesc().getId());
+ tblRefIds.add(lateralViewRef.getDesc().getId());
+ children.add(inputNode);
+ this.lateralViewRef = lateralViewRef;
+ }
+
+ @Override
+ public void init(Analyzer analyzer) throws UserException {
+ super.init(analyzer);
+ fnCallExpr = lateralViewRef.getFnExpr();
+ computeStats(analyzer);
+ }
+
+ @Override
+ protected void computeStats(Analyzer analyzer) {
+ super.computeStats(analyzer);
+ // TODO the cardinality = child cardinality * cardinality of list
column
+ cardinality = children.get(0).cardinality;
+ }
+
+ @Override
+ public String getNodeExplainString(String prefix, TExplainLevel
detailLevel) {
+ StringBuilder output = new StringBuilder();
+ output.append(prefix + "table function: ").append(fnCallExpr.toSql() +
"\n");
+ if (detailLevel == TExplainLevel.BRIEF) {
+ return output.toString();
+ }
+
+ if (!conjuncts.isEmpty()) {
+ output.append(prefix).append("PREDICATES: ").append(
+ getExplainString(conjuncts)).append("\n");
+ }
+ output.append(prefix).append(String.format("cardinality=%s",
cardinality)).append("\n");
+ return output.toString();
+ }
+
+ @Override
+ protected void toThrift(TPlanNode msg) {
+ msg.node_type = TPlanNodeType.TABLE_FUNCTION_NODE;
+ msg.table_function_node = new TTableFunctionNode();
+ msg.table_function_node.setFnCallExpr(fnCallExpr.treeToThrift());
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 4e9e0e9..13a5cf7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -160,6 +160,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_PARALLEL_OUTFILE =
"enable_parallel_outfile";
+ public static final String ENABLE_LATERAL_VIEW = "enable_lateral_view";
+
// session origin value
public Map<Field, String> sessionOriginValue = new HashMap<Field,
String>();
// check stmt is or not [select /*+ SET_VAR(...)*/ ...]
@@ -379,6 +381,9 @@ public class SessionVariable implements Serializable,
Writable {
@VariableMgr.VarAttr(name = CPU_RESOURCE_LIMIT)
public int cpuResourceLimit = -1;
+ @VariableMgr.VarAttr(name = ENABLE_LATERAL_VIEW, needForward = true)
+ public boolean enableLateralView = false;
+
public long getMaxExecMemByte() {
return maxExecMemByte;
}
@@ -787,6 +792,14 @@ public class SessionVariable implements Serializable,
Writable {
return enableParallelOutfile;
}
+ public boolean isEnableLateralView() {
+ return enableLateralView;
+ }
+
+ public void setEnableLateralView(boolean enableLateralView) {
+ this.enableLateralView = enableLateralView;
+ }
+
// Serialize to thrift object
// used for rest api
public TQueryOptions toThrift() {
diff --git a/fe/fe-core/src/main/jflex/sql_scanner.flex
b/fe/fe-core/src/main/jflex/sql_scanner.flex
index 42cb39c..f52f3a6 100644
--- a/fe/fe-core/src/main/jflex/sql_scanner.flex
+++ b/fe/fe-core/src/main/jflex/sql_scanner.flex
@@ -247,6 +247,7 @@ import org.apache.doris.qe.SqlModeHelper;
keywordMap.put("label", new Integer(SqlParserSymbols.KW_LABEL));
keywordMap.put("largeint", new Integer(SqlParserSymbols.KW_LARGEINT));
keywordMap.put("last", new Integer(SqlParserSymbols.KW_LAST));
+ keywordMap.put("lateral", new Integer(SqlParserSymbols.KW_LATERAL));
keywordMap.put("left", new Integer(SqlParserSymbols.KW_LEFT));
keywordMap.put("less", new Integer(SqlParserSymbols.KW_LESS));
keywordMap.put("level", new Integer(SqlParserSymbols.KW_LEVEL));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/planner/RuntimeFilterGeneratorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/planner/RuntimeFilterGeneratorTest.java
index 09fe44a..9c9816c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/planner/RuntimeFilterGeneratorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/planner/RuntimeFilterGeneratorTest.java
@@ -17,6 +17,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.UserException;
import org.apache.doris.common.jmockit.Deencapsulation;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TPartitionType;
@@ -43,7 +44,7 @@ public class RuntimeFilterGeneratorTest {
private ConnectContext connectContext;
@Before
- public void setUp() throws AnalysisException {
+ public void setUp() throws UserException {
Catalog catalog = Deencapsulation.newInstance(Catalog.class);
analyzer = new Analyzer(catalog, connectContext);
new Expectations() {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
new file mode 100644
index 0000000..1d59d15
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/planner/TableFunctionPlanTest.java
@@ -0,0 +1,199 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.planner;
+
+import org.apache.doris.analysis.CreateDbStmt;
+import org.apache.doris.analysis.CreateTableStmt;
+import org.apache.doris.catalog.Catalog;
+import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.utframe.UtFrameUtils;
+
+import org.apache.commons.io.FileUtils;
+
+import java.io.File;
+import java.util.UUID;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TableFunctionPlanTest {
+ private static String runningDir = "fe/mocked/TableFunctionPlanTest/" +
UUID.randomUUID().toString() + "/";
+ private static ConnectContext ctx;
+
+ @After
+ public void tearDown() throws Exception {
+ FileUtils.deleteDirectory(new File(runningDir));
+ }
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ UtFrameUtils.createDorisCluster(runningDir);
+ ctx = UtFrameUtils.createDefaultCtx();
+ ctx.getSessionVariable().setEnableLateralView(true);
+ String createDbStmtStr = "create database db1;";
+ CreateDbStmt createDbStmt = (CreateDbStmt)
UtFrameUtils.parseAndAnalyzeStmt(createDbStmtStr, ctx);
+ Catalog.getCurrentCatalog().createDb(createDbStmt);
+ // 3. create table tbl1
+ String createTblStmtStr = "create table db1.tbl1(k1 int, k2 varchar) "
+ + "DUPLICATE KEY(k1) distributed by hash(k1) buckets 3
properties('replication_num' = '1');";
+ CreateTableStmt createTableStmt = (CreateTableStmt)
UtFrameUtils.parseAndAnalyzeStmt(createTblStmtStr, ctx);
+ Catalog.getCurrentCatalog().createTable(createTableStmt);
+ }
+
+ // test planner
+ /* Case1 normal table function
+ select k1, e1 from table lateral view explode_split(k2, ",") tmp as e1;
+ */
+ @Test
+ public void normalTableFunction() throws Exception {
+ String sql = "desc verbose select k1, e1 from db1.tbl1 lateral view
explode_split(k2, \",\") tmp as e1;";
+ String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql,
true);
+ Assert.assertTrue(explainString.contains("1:TABLE FUNCTION NODE"));
+ Assert.assertTrue(explainString.contains("table function:
explode_split(`k2`, ',')"));
+ Assert.assertTrue(explainString.contains("tuple ids: 0 1"));
+ Assert.assertTrue(explainString.contains("TupleDescriptor{id=1,
tbl=tmp, byteSize=32, materialized=true}"));
+ Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1,
type=VARCHAR(*)}"));
+ }
+
+ /* Case2 without output explode column
+ select k1 from table lateral view explode_split(k2, ",") tmp as e1;
+ */
+ @Test
+ public void withoutOutputExplodeColumn() throws Exception {
+ String sql = "desc verbose select k1 from db1.tbl1 lateral view
explode_split(k2, \",\") tmp as e1;";
+ String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql,
true);
+ Assert.assertTrue(explainString.contains("OUTPUT EXPRS:`k1`"));
+ Assert.assertTrue(explainString.contains("1:TABLE FUNCTION NODE"));
+ Assert.assertTrue(explainString.contains("table function:
explode_split(`k2`, ',')"));
+ Assert.assertTrue(explainString.contains("tuple ids: 0 1"));
+ Assert.assertTrue(explainString.contains("TupleDescriptor{id=1,
tbl=tmp, byteSize=32, materialized=true}"));
+ Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1,
type=VARCHAR(*)}"));
+ }
+
+ /* Case3 group by explode column
+ select k1, e1, count(*) from table lateral view explode_split(k2, ",")
tmp as e1 group by k1 e1;
+ */
+ @Test
+ public void groupByExplodeColumn() throws Exception {
+ String sql = "desc verbose select k1, e1, count(*) from db1.tbl1
lateral view explode_split(k2, \",\") tmp as e1 "
+ + "group by k1, e1;";
+ String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql,
true);
+ // group by node with k1, e1
+ Assert.assertTrue(explainString.contains("2:AGGREGATE (update
finalize)"));
+ Assert.assertTrue(explainString.contains("group by: `k1`, `e1`"));
+ // table function node
+ Assert.assertTrue(explainString.contains("1:TABLE FUNCTION NODE"));
+ Assert.assertTrue(explainString.contains("table function:
explode_split(`k2`, ',')"));
+ Assert.assertTrue(explainString.contains("tuple ids: 0 1"));
+ Assert.assertTrue(explainString.contains("TupleDescriptor{id=1,
tbl=tmp, byteSize=32, materialized=true}"));
+ Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1,
type=VARCHAR(*)}"));
+ // group by tuple
+ Assert.assertTrue(explainString.contains("TupleDescriptor{id=2,
tbl=null, byteSize=32, materialized=true}"));
+ }
+
+ /* Case4 where explode column
+ select k1, e1 from table lateral view explode_split(k2, ",") tmp as e1
where e1 = "1";
+ */
+ @Test
+ public void whereExplodeColumn() throws Exception {
+ String sql = "desc verbose select k1, e1 from db1.tbl1 lateral view
explode_split(k2, \",\") tmp as e1 "
+ + "where e1='1'; ";
+ String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql,
true);
+ Assert.assertTrue(explainString.contains("1:TABLE FUNCTION NODE"));
+ Assert.assertTrue(explainString.contains("table function:
explode_split(`k2`, ',')"));
+ Assert.assertTrue(explainString.contains("PREDICATES: `e1` = '1'"));
+ Assert.assertTrue(explainString.contains("tuple ids: 0 1"));
+ Assert.assertTrue(explainString.contains("TupleDescriptor{id=1,
tbl=tmp, byteSize=32, materialized=true}"));
+ Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1,
type=VARCHAR(*)}"));
+ }
+
+ /* Case5 where normal column
+ select k1, e1 from table lateral view explode_split(k2, ",") tmp as e1
where k1 = 1;
+ */
+ @Test
+ public void whereNormalColumn() throws Exception {
+ String sql = "desc verbose select k1, e1 from db1.tbl1 lateral view
explode_split(k2, \",\") tmp as e1 "
+ + "where k1=1; ";
+ String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql,
true);
+ Assert.assertTrue(explainString.contains("1:TABLE FUNCTION NODE"));
+ Assert.assertTrue(explainString.contains("table function:
explode_split(`k2`, ',')"));
+ Assert.assertTrue(explainString.contains("tuple ids: 0 1"));
+ Assert.assertTrue(explainString.contains("TupleDescriptor{id=1,
tbl=tmp, byteSize=32, materialized=true}"));
+ Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e1,
type=VARCHAR(*)}"));
+ Assert.assertTrue(explainString.contains("0:OlapScanNode"));
+ Assert.assertTrue(explainString.contains("PREDICATES: `k1` = 1"));
+ }
+
+ /* Case6 multi lateral view
+ select k1, e1, e2 from table lateral view explode_split(k2, ",") tmp1 as
e1
+ lateral view explode_split(k2, ",") tmp2 as
e2;
+ */
+ @Test
+ public void testMultiLateralView() throws Exception {
+ String sql = "desc verbose select k1, e1, e2 from db1.tbl1 lateral
view explode_split(k2, \",\") tmp1 as e1"
+ + " lateral view explode_split(k2, \",\") tmp2 as e2;";
+ String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql,
true);
+ Assert.assertTrue(explainString.contains("2:TABLE FUNCTION NODE"));
+ Assert.assertTrue(explainString.contains("table function:
explode_split(`k2`, ',')"));
+ Assert.assertTrue(explainString.contains("tuple ids: 0 1 2"));
+ Assert.assertTrue(explainString.contains("1:TABLE FUNCTION NODE"));
+ Assert.assertTrue(explainString.contains("table function:
explode_split(`k2`, ',')"));
+ Assert.assertTrue(explainString.contains("tuple ids: 0 1"));
+ // lateral view 2 tuple
+ Assert.assertTrue(explainString.contains("TupleDescriptor{id=1,
tbl=tmp2, byteSize=32, materialized=true}"));
+ Assert.assertTrue(explainString.contains("SlotDescriptor{id=1, col=e2,
type=VARCHAR(*)}"));
+ // lateral view 1 tuple
+ Assert.assertTrue(explainString.contains("TupleDescriptor{id=2,
tbl=tmp1, byteSize=32, materialized=true}"));
+ Assert.assertTrue(explainString.contains("SlotDescriptor{id=2, col=e1,
type=VARCHAR(*)}"));
+ }
+
+ // test explode_split function
+ // k1 int ,k2 string
+ /* Case1 error param
+ select k1, e1 from table lateral view explode_split(k2) tmp as e1;
+ select k1, e1 from table lateral view explode_split(k1) tmp as e1;
+ select k1, e1 from table lateral view explode_split(k2, k2) tmp as e1;
+ */
+ @Test
+ public void errorParam() throws Exception {
+ String sql = "explain select k1, e1 from db1.tbl1 lateral view
explode_split(k2) tmp as e1;";
+ String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql);
+ Assert.assertTrue(explainString.contains("Doris only support
`explode_split(varchar, varchar)` table function"));
+
+ sql = "explain select k1, e1 from db1.tbl1 lateral view
explode_split(k1) tmp as e1;";
+ explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql);
+ Assert.assertTrue(explainString.contains("Doris only support
`explode_split(varchar, varchar)` table function"));
+
+ sql = "explain select k1, e1 from db1.tbl1 lateral view
explode_split(k1, k2) tmp as e1;";
+ explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql);
+ Assert.assertTrue(explainString.contains("Split separator of explode
must be a string const"));
+ }
+
+ /* Case2 table function in where stmt
+ select k1 from table where explode_split(k2, ",") = "1";
+ */
+ @Test
+ public void tableFunctionInWhere() throws Exception {
+ String sql = "explain select k1 from db1.tbl1 where explode_split(k2,
\",\");";
+ String explainString = UtFrameUtils.getSQLPlanOrErrorMsg(ctx, sql);
+ Assert.assertTrue(
+ explainString.contains("No matching function with signature:
explode_split(varchar(-1), varchar(-1))."));
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
index a7033ce..518e210 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/UtFrameUtils.java
@@ -244,12 +244,16 @@ public class UtFrameUtils {
}
public static String getSQLPlanOrErrorMsg(ConnectContext ctx, String
queryStr) throws Exception {
+ return getSQLPlanOrErrorMsg(ctx, queryStr, false);
+ }
+
+ public static String getSQLPlanOrErrorMsg(ConnectContext ctx, String
queryStr, boolean isVerbose) throws Exception {
ctx.getState().reset();
StmtExecutor stmtExecutor = new StmtExecutor(ctx, queryStr);
stmtExecutor.execute();
if (ctx.getState().getStateType() != QueryState.MysqlStateType.ERR) {
Planner planner = stmtExecutor.planner();
- return planner.getExplainString(planner.getFragments(), new
ExplainOptions(false, false));
+ return planner.getExplainString(planner.getFragments(), new
ExplainOptions(isVerbose, false));
} else {
return ctx.getState().getErrorMessage();
}
diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift
index 0a90997..c76ed33 100644
--- a/gensrc/thrift/PlanNodes.thrift
+++ b/gensrc/thrift/PlanNodes.thrift
@@ -51,6 +51,7 @@ enum TPlanNodeType {
INTERSECT_NODE,
EXCEPT_NODE,
ODBC_SCAN_NODE,
+ TABLE_FUNCTION_NODE,
}
// phases of an execution node
@@ -652,6 +653,10 @@ struct TOlapRewriteNode {
3: required Types.TTupleId output_tuple_id
}
+struct TTableFunctionNode {
+ 1: required Exprs.TExpr fnCallExpr
+}
+
// This contains all of the information computed by the plan as part of the
resource
// profile that is needed by the backend to execute.
struct TBackendResourceProfile {
@@ -772,6 +777,8 @@ struct TPlanNode {
36: optional list<TRuntimeFilterDesc> runtime_filters
40: optional Exprs.TExpr vconjunct
+
+ 41: optional TTableFunctionNode table_function_node
}
// A flattened representation of a tree of PlanNodes, obtained by depth-first
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
