This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new c51f20b Disable Bitmap or Hll type in keys or in values with
incorrect agg-type (#3768)
c51f20b is described below
commit c51f20bb7a2436954af70fdaccb81167f1c92183
Author: Xiang Wei <[email protected]>
AuthorDate: Sat Jun 6 11:36:06 2020 +0800
Disable Bitmap or Hll type in keys or in values with incorrect agg-type
(#3768)
Bitmap and Hll type can not be used with incorrect aggregate functions,
which will cause to BE crush.
Add some logical checks in FE's ColumnDef#analyze to avoid creating tables
or changing schemas incorrectly.
Keys never be bitmap or hll type
values with bitmap or hll type have to be associated with bitmap_union or
hll_union
---
.../java/org/apache/doris/analysis/ColumnDef.java | 10 +++
.../org/apache/doris/analysis/CreateTableStmt.java | 14 +---
.../org/apache/doris/catalog/AggregateType.java | 15 ++--
.../apache/doris/analysis/CreateTableStmtTest.java | 83 ++++++++++++++++++++++
4 files changed, 103 insertions(+), 19 deletions(-)
diff --git a/fe/src/main/java/org/apache/doris/analysis/ColumnDef.java
b/fe/src/main/java/org/apache/doris/analysis/ColumnDef.java
index 5227b2e..d51e6ed 100644
--- a/fe/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -131,6 +131,16 @@ public class ColumnDef {
Type type = typeDef.getType();
+ // disable Bitmap Hll type in keys, values without aggregate function.
+ if (type.isBitmapType() || type.isHllType()) {
+ if (isKey) {
+ throw new AnalysisException("Key column can not set bitmap or
hll type:" + name);
+ }
+ if (aggregateType == null) {
+ throw new AnalysisException("Bitmap and hll type have to use
aggregate function" + name);
+ }
+ }
+
// A column is a key column if and only if isKey is true.
// aggregateType == null does not mean that this is a key column,
// because when creating a UNIQUE KEY table, aggregateType is implicit.
diff --git a/fe/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
b/fe/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 526cfd0..7a87055 100644
--- a/fe/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -322,24 +322,12 @@ public class CreateTableStmt extends DdlStmt {
columnDef.analyze(engineName.equals("olap"));
if (columnDef.getType().isHllType()) {
- if (columnDef.isKey()) {
- throw new AnalysisException("HLL can't be used as keys, " +
- "please specify the aggregation type HLL_UNION");
- }
hasHll = true;
}
- if (columnDef.getType().isBitmapType()) {
- if (columnDef.isKey()) {
- throw new AnalysisException("BITMAP can't be used as keys,
");
- }
- }
if (columnDef.getAggregateType() == BITMAP_UNION) {
- if (columnDef.isKey()) {
- throw new AnalysisException("Key column can't has the
BITMAP_UNION aggregation type");
- }
- hasBitmap = true;
+ hasBitmap = columnDef.getType().isBitmapType();
}
if (!columnSet.add(columnDef.getName())) {
diff --git a/fe/src/main/java/org/apache/doris/catalog/AggregateType.java
b/fe/src/main/java/org/apache/doris/catalog/AggregateType.java
index c39dac2..bce3c14 100644
--- a/fe/src/main/java/org/apache/doris/catalog/AggregateType.java
+++ b/fe/src/main/java/org/apache/doris/catalog/AggregateType.java
@@ -85,11 +85,14 @@ public enum AggregateType {
compatibilityMap.put(MAX, EnumSet.copyOf(primitiveTypeList));
primitiveTypeList.clear();
- compatibilityMap.put(REPLACE, EnumSet.allOf(PrimitiveType.class));
+ // all types except bitmap and hll.
+ EnumSet<PrimitiveType> exc_bitmap_hll =
EnumSet.allOf(PrimitiveType.class);
+ exc_bitmap_hll.remove(PrimitiveType.BITMAP);
+ exc_bitmap_hll.remove(PrimitiveType.HLL);
+ compatibilityMap.put(REPLACE, EnumSet.copyOf(exc_bitmap_hll));
+
+ compatibilityMap.put(REPLACE_IF_NOT_NULL,
EnumSet.copyOf(exc_bitmap_hll));
- primitiveTypeList.clear();
- compatibilityMap.put(REPLACE_IF_NOT_NULL,
EnumSet.allOf(PrimitiveType.class));
-
primitiveTypeList.clear();
primitiveTypeList.add(PrimitiveType.HLL);
compatibilityMap.put(HLL_UNION, EnumSet.copyOf(primitiveTypeList));
@@ -97,8 +100,8 @@ public enum AggregateType {
primitiveTypeList.clear();
primitiveTypeList.add(PrimitiveType.BITMAP);
compatibilityMap.put(BITMAP_UNION, EnumSet.copyOf(primitiveTypeList));
-
- compatibilityMap.put(NONE, EnumSet.allOf(PrimitiveType.class));
+
+ compatibilityMap.put(NONE, EnumSet.copyOf(exc_bitmap_hll));
}
private final String sqlName;
diff --git
a/fe/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java
b/fe/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java
index 3a25ba3..cf52ee6 100644
--- a/fe/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java
+++ b/fe/src/test/java/org/apache/doris/analysis/CreateTableStmtTest.java
@@ -18,6 +18,7 @@
package org.apache.doris.analysis;
import mockit.Expectations;
+import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
@@ -31,7 +32,9 @@ import com.google.common.collect.Lists;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -55,6 +58,8 @@ public class CreateTableStmtTest {
private PaloAuth auth;
@Mocked
private ConnectContext ctx;
+ @Rule
+ public ExpectedException expectedEx = ExpectedException.none();
// set default db is 'db1'
// table name is table1
@@ -163,4 +168,82 @@ public class CreateTableStmtTest {
new RandomDistributionDesc(10), null, null, "");
stmt.analyze(analyzer);
}
+
+
+ @Test
+ public void testBmpHllKey() throws Exception {
+
+ ColumnDef bitmap = new ColumnDef("col3", new
TypeDef(ScalarType.createType(PrimitiveType.BITMAP)));
+ cols.add(bitmap);
+ colsName.add("col3");
+
+ CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb,
cols, "olap",
+ new KeysDesc(KeysType.AGG_KEYS, colsName), null,
+ new RandomDistributionDesc(10), null, null, "");
+ expectedEx.expect(AnalysisException.class);
+ expectedEx.expectMessage("Key column can not set bitmap or hll
type:col3");
+ stmt.analyze(analyzer);
+
+ cols.remove(bitmap);
+
+ ColumnDef hll = new ColumnDef("col3", new
TypeDef(ScalarType.createType(PrimitiveType.HLL)));
+ cols.add(hll);
+ stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap",
+ new KeysDesc(KeysType.AGG_KEYS, colsName), null,
+ new RandomDistributionDesc(10), null, null, "");
+ expectedEx.expect(AnalysisException.class);
+ expectedEx.expectMessage("Key column can not set bitmap or hll
type:col3");
+ stmt.analyze(analyzer);
+ }
+
+ @Test
+ public void testBmpHllNoAggTab() throws Exception {
+ ColumnDef bitmap = new ColumnDef("col3", new
TypeDef(ScalarType.createType(PrimitiveType.BITMAP)));
+ cols.add(bitmap);
+ CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb,
cols, "olap",
+ new KeysDesc(KeysType.DUP_KEYS, colsName), null,
+ new RandomDistributionDesc(10), null, null, "");
+ expectedEx.expect(AnalysisException.class);
+ expectedEx.expectMessage("Aggregate type `col3` bitmap NONE NOT NULL
COMMENT \"\" is not compatible with primitive type bitmap");
+ stmt.analyze(analyzer);
+
+ cols.remove(bitmap);
+ ColumnDef hll = new ColumnDef("col3", new
TypeDef(ScalarType.createType(PrimitiveType.HLL)));
+ cols.add(hll);
+ stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap",
+ new KeysDesc(KeysType.DUP_KEYS, colsName), null,
+ new RandomDistributionDesc(10), null, null, "");
+ expectedEx.expect(AnalysisException.class);
+ expectedEx.expectMessage("Aggregate type `col3` hll NONE NOT NULL
COMMENT \"\" is not compatible with primitive type hll");
+ stmt.analyze(analyzer);
+ }
+
+ @Test
+ public void testBmpHllIncAgg() throws Exception {
+ ColumnDef bitmap = new ColumnDef("col3", new
TypeDef(ScalarType.createType(PrimitiveType.BITMAP)));
+ bitmap.setAggregateType(AggregateType.SUM);
+
+ cols.add(bitmap);
+ CreateTableStmt stmt = new CreateTableStmt(false, false, tblNameNoDb,
cols, "olap",
+ new KeysDesc(KeysType.AGG_KEYS, colsName), null,
+ new RandomDistributionDesc(10), null, null, "");
+
+ expectedEx.expect(AnalysisException.class);
+ expectedEx.expectMessage(String.format("Aggregate type %s is not
compatible with primitive type %s",
+ bitmap.toString(), bitmap.getTypeDef().getType().toSql()));
+ stmt.analyze(analyzer);
+
+ cols.remove(bitmap);
+ ColumnDef hll = new ColumnDef("col3", new
TypeDef(ScalarType.createType(PrimitiveType.HLL)));
+ hll.setAggregateType(AggregateType.SUM);
+ cols.add(hll);
+ stmt = new CreateTableStmt(false, false, tblNameNoDb, cols, "olap",
+ new KeysDesc(KeysType.AGG_KEYS, colsName), null,
+ new RandomDistributionDesc(10), null, null, "");
+
+ expectedEx.expect(AnalysisException.class);
+ expectedEx.expectMessage(String.format("Aggregate type %s is not
compatible with primitive type %s",
+ hll.toString(), hll.getTypeDef().getType().toSql()));
+ stmt.analyze(analyzer);
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
