This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 76a04de [MV] Input correct keys type of index meta when `Add
Partition` (#4408)
76a04de is described below
commit 76a04de6c44ab959d3f5c32e7711b5e429beb98d
Author: EmmyMiao87 <[email protected]>
AuthorDate: Fri Aug 21 10:42:41 2020 +0800
[MV] Input correct keys type of index meta when `Add Partition` (#4408)
Define Expr will not serialized in Column `toThrift`.
1. When adding partition, different indexes should use their own keys type
instead of using the keys type of base table uniformly.
`
2. There are two kinds of define expr in Column , one is analyzed, and the
other is not analyzed.
Currently, analyzed define expr is only used when creating materialized
views, so the define expr in RollupJob must be analyzed.
In other cases, such as define expr in `MaterializedIndexMeta`, it may not
be analyzed after being relayed.
When executing the load, the analyzed define expr (such as
to_bitmap(cast(k1, varchar))) will not be analyzed again.
Only a cast function will be added to the inner layer(such as
to_bitmap(cast(cast(k1 ,int), varchar))) which is analyzed too.
The define expr that has not been analyzed (such as cast(k1, varchar)) will
be analyzed when executing the load.
---
fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java | 2 +-
.../org/apache/doris/analysis/CreateMaterializedViewStmt.java | 2 --
fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java | 8 ++------
fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java | 9 ++++++---
4 files changed, 9 insertions(+), 12 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
index 3094288..4ee4741 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CastExpr.java
@@ -72,7 +72,7 @@ public class CastExpr extends Expr {
/**
* Copy c'tor used in clone().
*/
- protected CastExpr(TypeDef targetTypeDef, Expr e) {
+ public CastExpr(TypeDef targetTypeDef, Expr e) {
Preconditions.checkNotNull(targetTypeDef);
Preconditions.checkNotNull(e);
this.targetTypeDef = targetTypeDef;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
index 7e73bcf..49fffa9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
@@ -433,7 +433,6 @@ public class CreateMaterializedViewStmt extends DdlStmt {
case FunctionSet.BITMAP_UNION:
if (functionCallExpr.getChild(0) instanceof
FunctionCallExpr) {
CastExpr castExpr = new CastExpr(new
TypeDef(Type.VARCHAR), baseSlotRef);
- castExpr.setImplicit(true);
List<Expr> params = Lists.newArrayList();
params.add(castExpr);
FunctionCallExpr defineExpr = new
FunctionCallExpr(FunctionSet.TO_BITMAP, params);
@@ -445,7 +444,6 @@ public class CreateMaterializedViewStmt extends DdlStmt {
case FunctionSet.HLL_UNION:
if (functionCallExpr.getChild(0) instanceof
FunctionCallExpr) {
CastExpr castExpr = new CastExpr(new
TypeDef(Type.VARCHAR), baseSlotRef);
- castExpr.setImplicit(true);
List<Expr> params = Lists.newArrayList();
params.add(castExpr);
FunctionCallExpr defineExpr = new
FunctionCallExpr(FunctionSet.HLL_HASH, params);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index be45a50..54ebcf4 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -3173,7 +3173,6 @@ public class Catalog {
olapTable.getBaseIndexId(),
partitionId, partitionName,
indexIdToMeta,
- olapTable.getKeysType(),
distributionInfo,
dataProperty.getStorageMedium(),
singlePartitionDesc.getReplicationNum(),
@@ -3395,7 +3394,6 @@ public class Catalog {
private Partition createPartitionWithIndices(String clusterName, long
dbId, long tableId,
long baseIndexId, long
partitionId, String partitionName,
Map<Long,
MaterializedIndexMeta> indexIdToMeta,
- KeysType keysType,
DistributionInfo
distributionInfo,
TStorageMedium storageMedium,
short replicationNum,
@@ -3453,6 +3451,7 @@ public class Catalog {
short shortKeyColumnCount = indexMeta.getShortKeyColumnCount();
TStorageType storageType = indexMeta.getStorageType();
List<Column> schema = indexMeta.getSchema();
+ KeysType keysType = indexMeta.getKeysType();
int totalTaskNum = index.getTablets().size() * replicationNum;
MarkedCountDownLatch<Long, Long> countDownLatch = new
MarkedCountDownLatch<Long, Long>(totalTaskNum);
AgentBatchTask batchTask = new AgentBatchTask();
@@ -3744,7 +3743,6 @@ public class Catalog {
olapTable.getId(), olapTable.getBaseIndexId(),
partitionId, partitionName,
olapTable.getIndexIdToMeta(),
- keysType,
distributionInfo,
partitionInfo.getDataProperty(partitionId).getStorageMedium(),
partitionInfo.getReplicationNum(partitionId),
@@ -3773,8 +3771,7 @@ public class Catalog {
DataProperty dataProperty =
rangePartitionInfo.getDataProperty(entry.getValue());
Partition partition =
createPartitionWithIndices(db.getClusterName(), db.getId(), olapTable.getId(),
olapTable.getBaseIndexId(), entry.getValue(),
entry.getKey(),
- olapTable.getIndexIdToMeta(),
- keysType, distributionInfo,
+ olapTable.getIndexIdToMeta(), distributionInfo,
dataProperty.getStorageMedium(),
partitionInfo.getReplicationNum(entry.getValue()),
versionInfo, bfColumns, bfFpp,
@@ -6286,7 +6283,6 @@ public class Catalog {
db.getId(), copiedTbl.getId(),
copiedTbl.getBaseIndexId(),
newPartitionId, entry.getKey(),
copiedTbl.getIndexIdToMeta(),
- copiedTbl.getKeysType(),
copiedTbl.getDefaultDistributionInfo(),
copiedTbl.getPartitionInfo().getDataProperty(oldPartitionId).getStorageMedium(),
copiedTbl.getPartitionInfo().getReplicationNum(oldPartitionId),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 63aa2c5..f0b0a3c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -72,6 +72,9 @@ public class Column implements Writable {
private String comment;
@SerializedName(value = "stats")
private ColumnStats stats; // cardinality and selectivity etc.
+ // Define expr may exist in two forms, one is analyzed, and the other is
not analyzed.
+ // Currently, analyzed define expr is only used when creating materialized
views, so the define expr in RollupJob must be analyzed.
+ // In other cases, such as define expr in `MaterializedIndexMeta`, it may
not be analyzed after being relayed.
private Expr defineExpr; // use to define column in materialize view
public Column() {
@@ -259,9 +262,9 @@ public class Column implements Writable {
tColumn.setIs_key(this.isKey);
tColumn.setIs_allow_null(this.isAllowNull);
tColumn.setDefault_value(this.defaultValue);
- if (this.defineExpr != null) {
- tColumn.setDefine_expr(this.defineExpr.treeToThrift());
- }
+ // The define expr does not need to be serialized here for now.
+ // At present, only serialized(analyzed) define expr is directly used
when creating a materialized view.
+ // It will not be used here, but through another structure
`TAlterMaterializedViewParam`.
return tColumn;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
