This is an automated email from the ASF dual-hosted git repository.
lingmiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2608f83 [WIP] Add define expr for column (#3651)
2608f83 is described below
commit 2608f83bdce7df7a0df103e0492be0b92455659d
Author: HangyuanLiu <[email protected]>
AuthorDate: Mon May 25 11:00:29 2020 +0800
[WIP] Add define expr for column (#3651)
In the materialized view 2.0 the define expr should be set in column.
For example, the to_bitmap function on integer should be define in mv
column.
```
create materialized view mv as select bitmap_union(to_bitmap(k1)) from
table.
```
The meta of mv as following:
column name: __doris_materialized_view_bitmap_k1
column aggregate type: bitmap_union
column define exrp: to_bitmap(k1)
The is WIP pr for materialized view 2.0.
#3344
---
.../org/apache/doris/alter/MaterializedViewHandler.java | 16 ++++++++++++++++
.../doris/analysis/CreateMaterializedViewStmt.java | 3 +++
.../java/org/apache/doris/analysis/DescribeStmt.java | 3 +++
.../java/org/apache/doris/analysis/MVColumnItem.java | 9 +++++++++
fe/src/main/java/org/apache/doris/catalog/Column.java | 17 +++++++++++++++++
gensrc/thrift/AgentService.thrift | 2 ++
6 files changed, 50 insertions(+)
diff --git
a/fe/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
b/fe/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
index 0d3c792..e4827f1 100644
--- a/fe/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
+++ b/fe/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
@@ -41,6 +41,7 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
import org.apache.doris.catalog.TabletMeta;
+import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
@@ -88,6 +89,7 @@ import java.util.concurrent.ConcurrentHashMap;
public class MaterializedViewHandler extends AlterHandler {
private static final Logger LOG =
LogManager.getLogger(MaterializedViewHandler.class);
public static final String NEW_STORAGE_FORMAT_INDEX_NAME_PREFIX = "__v2_";
+ public static final String MATERIALIZED_VIEW_NAME_PRFIX =
"__doris_materialized_view_";
public MaterializedViewHandler() {
super("materialized view");
@@ -459,6 +461,20 @@ public class MaterializedViewHandler extends AlterHandler {
Column newMVColumn = new Column(baseColumn);
newMVColumn.setIsKey(mvColumnItem.isKey());
newMVColumn.setAggregationType(mvAggregationType,
mvColumnItem.isAggregationTypeImplicit());
+ if (mvColumnItem.getDefineExpr() != null) {
+ if (mvAggregationType.equals(AggregateType.BITMAP_UNION)) {
+ newMVColumn.setType(Type.BITMAP);
+ newMVColumn.setName(MATERIALIZED_VIEW_NAME_PRFIX +
"bitmap_" + baseColumn.getName());
+ } else if (mvAggregationType.equals(AggregateType.HLL_UNION)){
+ newMVColumn.setType(Type.HLL);
+ newMVColumn.setName(MATERIALIZED_VIEW_NAME_PRFIX + "hll_"
+ baseColumn.getName());
+ } else {
+ throw new DdlException("The define expr of column is only
support bitmap_union or hll_union");
+ }
+ newMVColumn.setIsKey(false);
+ newMVColumn.setIsAllowNull(false);
+ newMVColumn.setDefineExpr(mvColumnItem.getDefineExpr());
+ }
newMVColumns.add(newMVColumn);
}
return newMVColumns;
diff --git
a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
index d01310f..5690be3 100644
--- a/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
+++ b/fe/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
@@ -167,6 +167,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
} else if (selectListItem.getExpr() instanceof FunctionCallExpr) {
FunctionCallExpr functionCallExpr = (FunctionCallExpr)
selectListItem.getExpr();
String functionName =
functionCallExpr.getFnName().getFunction();
+ Expr defineExpr = null;
// TODO(ml): support REPLACE, REPLACE_IF_NOT_NULL only for
aggregate table, HLL_UNION, BITMAP_UNION
if (!functionName.equalsIgnoreCase("sum")
&& !functionName.equalsIgnoreCase("min")
@@ -174,6 +175,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
throw new AnalysisException("The materialized view only
support the sum, min and max aggregate "
+ "function. Error
function: " + functionCallExpr.toSqlImpl());
}
+
Preconditions.checkState(functionCallExpr.getChildren().size()
== 1);
Expr functionChild0 = functionCallExpr.getChild(0);
SlotRef slotRef;
@@ -200,6 +202,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
// TODO(ml): support different type of column, int ->
bigint(sum)
MVColumnItem mvColumnItem = new MVColumnItem(columnName);
mvColumnItem.setAggregationType(AggregateType.valueOf(functionName.toUpperCase()),
false);
+ mvColumnItem.setDefineExpr(defineExpr);
mvColumnItemList.add(mvColumnItem);
}
}
diff --git a/fe/src/main/java/org/apache/doris/analysis/DescribeStmt.java
b/fe/src/main/java/org/apache/doris/analysis/DescribeStmt.java
index c8ffd35..ca350f1 100644
--- a/fe/src/main/java/org/apache/doris/analysis/DescribeStmt.java
+++ b/fe/src/main/java/org/apache/doris/analysis/DescribeStmt.java
@@ -163,6 +163,9 @@ public class DescribeStmt extends ShowStmt {
if (bfColumns != null &&
bfColumns.contains(column.getName())) {
extras.add("BLOOM_FILTER");
}
+ if (column.getDefineExpr() != null) {
+
extras.add(column.getDefineExpr().toSql().toUpperCase());
+ }
String extraStr = StringUtils.join(extras, ",");
List<String> row = Arrays.asList("",
diff --git a/fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java
b/fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java
index a07c82a..a78e216 100644
--- a/fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java
+++ b/fe/src/main/java/org/apache/doris/analysis/MVColumnItem.java
@@ -30,6 +30,7 @@ public class MVColumnItem {
private boolean isKey;
private AggregateType aggregationType;
private boolean isAggregationTypeImplicit;
+ private Expr defineExpr;
public MVColumnItem(String name) {
this.name = name;
@@ -59,4 +60,12 @@ public class MVColumnItem {
public boolean isAggregationTypeImplicit() {
return isAggregationTypeImplicit;
}
+
+ public Expr getDefineExpr() {
+ return defineExpr;
+ }
+
+ public void setDefineExpr(Expr defineExpr) {
+ this.defineExpr = defineExpr;
+ }
}
diff --git a/fe/src/main/java/org/apache/doris/catalog/Column.java
b/fe/src/main/java/org/apache/doris/catalog/Column.java
index 4a1797a..d1ce8a6 100644
--- a/fe/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/src/main/java/org/apache/doris/catalog/Column.java
@@ -18,6 +18,7 @@
package org.apache.doris.catalog;
import org.apache.doris.alter.SchemaChangeHandler;
+import org.apache.doris.analysis.Expr;
import org.apache.doris.common.CaseSensibility;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeMetaVersion;
@@ -66,6 +67,7 @@ public class Column implements Writable {
private String comment;
@SerializedName(value = "stats")
private ColumnStats stats; // cardinality and selectivity etc.
+ private Expr defineExpr; //use to define materialize view
public Column() {
this.name = "";
@@ -157,6 +159,10 @@ public class Column implements Writable {
public Type getType() { return
ScalarType.createType(type.getPrimitiveType()); }
+ public void setType(Type type) {
+ this.type = type;
+ }
+
public Type getOriginType() { return type; }
public int getStrLen() { return ((ScalarType) type).getLength(); }
@@ -240,6 +246,9 @@ public class Column implements Writable {
tColumn.setIs_key(this.isKey);
tColumn.setIs_allow_null(this.isAllowNull);
tColumn.setDefault_value(this.defaultValue);
+ if (this.defineExpr != null) {
+ tColumn.setDefine_expr(this.defineExpr.treeToThrift());
+ }
return tColumn;
}
@@ -310,6 +319,14 @@ public class Column implements Writable {
return colName;
}
+ public Expr getDefineExpr() {
+ return defineExpr;
+ }
+
+ public void setDefineExpr(Expr expr) {
+ defineExpr = expr;
+ }
+
public String toSql() {
StringBuilder sb = new StringBuilder();
sb.append("`").append(name).append("` ");
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index f9d9208..a7fe637 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -22,6 +22,7 @@ include "Status.thrift"
include "Types.thrift"
include "PaloInternalService.thrift"
include "Descriptors.thrift"
+include "Exprs.thrift"
struct TColumn {
1: required string column_name
@@ -31,6 +32,7 @@ struct TColumn {
5: optional bool is_allow_null
6: optional string default_value
7: optional bool is_bloom_filter_column
+ 8: optional Exprs.TExpr define_expr
}
struct TTabletSchema {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
