This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new ca5a4f6d35a [Fix](Variant) variant should not implicit be short key
column when create mv (#46539)
ca5a4f6d35a is described below
commit ca5a4f6d35ab5018e18592e10f295316b78b11b7
Author: lihangyu <[email protected]>
AuthorDate: Thu Jan 9 08:15:22 2025 +0800
[Fix](Variant) variant should not implicit be short key column when create
mv (#46539)
cherry-pick from #46444
---
.../main/java/org/apache/doris/catalog/Type.java | 12 +++++++++
.../doris/alter/MaterializedViewHandler.java | 3 +--
.../doris/analysis/CreateMaterializedViewStmt.java | 6 +++--
.../org/apache/doris/analysis/CreateTableStmt.java | 11 +-------
.../main/java/org/apache/doris/catalog/Env.java | 2 +-
.../trees/plans/commands/info/CreateMTMVInfo.java | 8 +++---
.../trees/plans/commands/info/CreateTableInfo.java | 3 +--
regression-test/data/variant_p0/mv/multi_slot.out | 3 +++
.../test_create_mv_complex_type.groovy | 8 +++---
.../rollup_p0/test_materialized_view_array.groovy | 2 +-
.../rollup_p0/test_materialized_view_struct.groovy | 2 +-
.../suites/variant_p0/mv/multi_slot.groovy | 30 ++++++++++++++++++++--
12 files changed, 61 insertions(+), 29 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index 0d052d900b5..8761ffa9e8d 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -381,6 +381,18 @@ public abstract class Type {
return false;
}
+ /**
+ * Return true if this type can be as short key
+ */
+ public boolean couldBeShortKey() {
+ return !(isFloatingPointType()
+ || getPrimitiveType() == PrimitiveType.STRING
+ || isJsonbType()
+ || isComplexType()
+ || isObjectStored()
+ || isVariantType());
+ }
+
/**
* The output of this is stored directly in the hive metastore as the
column type.
* The string must match exactly.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
index b50bec9cb02..9623ef562d2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
@@ -767,10 +767,9 @@ public class MaterializedViewHandler extends AlterHandler {
break;
}
}
- if (column.getType().isFloatingPointType()) {
+ if (!column.getType().couldBeShortKey()) {
break;
}
-
column.setIsKey(true);
if (column.getType().getPrimitiveType() ==
PrimitiveType.VARCHAR) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
index ace179410d1..3d98a1070e5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateMaterializedViewStmt.java
@@ -468,7 +468,7 @@ public class CreateMaterializedViewStmt extends DdlStmt {
}
break;
}
- if (column.getType().isFloatingPointType()) {
+ if (!column.getType().couldBeShortKey()) {
break;
}
if (column.getType().getPrimitiveType() ==
PrimitiveType.VARCHAR) {
@@ -479,7 +479,9 @@ public class CreateMaterializedViewStmt extends DdlStmt {
column.setIsKey(true);
}
if (theBeginIndexOfValue == 0) {
- throw new AnalysisException("The first column could not be
float or double type, use decimal instead");
+ throw new AnalysisException(
+ "The first column could not be float, double or complex "
+ + "type like array, struct, map, json, variant.");
}
// supply value
for (; theBeginIndexOfValue < mvColumnItemList.size();
theBeginIndexOfValue++) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
index 3dccc18730a..10939259a75 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateTableStmt.java
@@ -366,16 +366,7 @@ public class CreateTableStmt extends DdlStmt {
}
break;
}
- if (columnDef.getType().isFloatingPointType()) {
- break;
- }
- if (columnDef.getType().getPrimitiveType() ==
PrimitiveType.STRING) {
- break;
- }
- if (columnDef.getType().getPrimitiveType() ==
PrimitiveType.JSONB) {
- break;
- }
- if (columnDef.getType().isComplexType()) {
+ if (!columnDef.getType().couldBeShortKey()) {
break;
}
if (columnDef.getType().getPrimitiveType() ==
PrimitiveType.VARCHAR) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
index 70e4a0f57d5..971fcf407e8 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
@@ -4437,7 +4437,7 @@ public class Env {
}
break;
}
- if (column.getType().isFloatingPointType()) {
+ if (!column.getType().couldBeShortKey()) {
break;
}
if (column.getDataType() == PrimitiveType.VARCHAR) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
index c4b6286d193..64c577f7f60 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java
@@ -312,10 +312,10 @@ public class CreateMTMVInfo {
}
break;
}
- if (type.isFloatLikeType() || type.isStringType() ||
type.isJsonType()
- || catalogType.isComplexType() || type.isBitmapType()
|| type.isHllType()
- || type.isQuantileStateType() || type.isJsonType() ||
type.isStructType()
- || column.getAggType() != null ||
type.isVariantType()) {
+ if (column.getAggType() != null) {
+ break;
+ }
+ if (!catalogType.couldBeShortKey()) {
break;
}
keys.add(column.getName());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
index 09c44626e78..58ea028ecc1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateTableInfo.java
@@ -348,8 +348,7 @@ public class CreateTableInfo {
}
break;
}
- if (type.isFloatLikeType() || type.isStringType()
|| type.isJsonType()
- || catalogType.isComplexType() ||
catalogType.isVariantType()) {
+ if (!catalogType.couldBeShortKey()) {
break;
}
keys.add(column.getName());
diff --git a/regression-test/data/variant_p0/mv/multi_slot.out
b/regression-test/data/variant_p0/mv/multi_slot.out
index 2499e282895..fd2b064b069 100644
--- a/regression-test/data/variant_p0/mv/multi_slot.out
+++ b/regression-test/data/variant_p0/mv/multi_slot.out
@@ -41,3 +41,6 @@
3 \N
5 \N
+-- !sql --
+2021-01-01T11:11:11 "http://xxx.xxx.xxx"; 12
+
diff --git
a/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
b/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
index 1e07d69ba82..2ce17abcc0b 100644
---
a/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
+++
b/regression-test/suites/mv_p0/test_create_mv_complex_type/test_create_mv_complex_type.groovy
@@ -47,7 +47,7 @@ suite ("create_mv_complex_type") {
sql """create materialized view mv as select c_jsonb, c_int from
base_table;"""
success = true
} catch (Exception e) {
- assertTrue(e.getMessage().contains("not support to create materialized
view"), e.getMessage())
+ assertTrue(e.getMessage().contains("The first column could not be"),
e.getMessage())
}
assertFalse(success)
@@ -65,7 +65,7 @@ suite ("create_mv_complex_type") {
sql """create materialized view mv as select c_array, c_int from
base_table;"""
success = true
} catch (Exception e) {
- assertTrue(e.getMessage().contains("not support to create materialized
view"), e.getMessage())
+ assertTrue(e.getMessage().contains("The first column could not be"),
e.getMessage())
}
assertFalse(success)
@@ -83,7 +83,7 @@ suite ("create_mv_complex_type") {
sql """create materialized view mv as select c_map, c_int from
base_table;"""
success = true
} catch (Exception e) {
- assertTrue(e.getMessage().contains("not support to create materialized
view"), e.getMessage())
+ assertTrue(e.getMessage().contains("The first column could not be"),
e.getMessage())
}
assertFalse(success)
@@ -101,7 +101,7 @@ suite ("create_mv_complex_type") {
sql """create materialized view mv as select c_struct, c_int from
base_table;"""
success = true
} catch (Exception e) {
- assertTrue(e.getMessage().contains("not support to create materialized
view"), e.getMessage())
+ assertTrue(e.getMessage().contains("The first column could not be"),
e.getMessage())
}
assertFalse(success)
diff --git
a/regression-test/suites/rollup_p0/test_materialized_view_array.groovy
b/regression-test/suites/rollup_p0/test_materialized_view_array.groovy
index dad735a76ce..52b8a32740d 100644
--- a/regression-test/suites/rollup_p0/test_materialized_view_array.groovy
+++ b/regression-test/suites/rollup_p0/test_materialized_view_array.groovy
@@ -67,7 +67,7 @@ suite("test_materialized_view_array", "rollup") {
create_test_table.call(tableName)
test {
sql "CREATE MATERIALIZED VIEW idx AS select k2,k1, k3, k4, k5 from
${tableName}"
- exception "errCode = 2, detailMessage = The ARRAY column[`mv_k2`
array<smallint> NULL] not support to create materialized view"
+ exception "errCode = 2, detailMessage = The first column could not
be float, double or complex type like array, struct, map, json, variant"
}
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git
a/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy
b/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy
index 61e8415cacc..2a05b895384 100644
--- a/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy
+++ b/regression-test/suites/rollup_p0/test_materialized_view_struct.groovy
@@ -59,7 +59,7 @@ suite("test_materialized_view_struct", "rollup") {
create_test_table.call(tableName)
test {
sql "CREATE MATERIALIZED VIEW idx AS select k2,k1, k3, k4, k5 from
${tableName}"
- exception "errCode = 2, detailMessage = The STRUCT column[`mv_k2`
struct<f1:smallint> NULL] not support to create materialized view"
+ exception "errCode = 2, detailMessage = The first column could not
be float, double or complex type like array, struct, map, json, variant."
}
} finally {
try_sql("DROP TABLE IF EXISTS ${tableName}")
diff --git a/regression-test/suites/variant_p0/mv/multi_slot.groovy
b/regression-test/suites/variant_p0/mv/multi_slot.groovy
index 98b8f7f549e..691eff06dec 100644
--- a/regression-test/suites/variant_p0/mv/multi_slot.groovy
+++ b/regression-test/suites/variant_p0/mv/multi_slot.groovy
@@ -50,8 +50,7 @@ suite ("multi_slot") {
order_qt_select_star "select abs(cast(v['k1'] as int))+cast(v['k2'] as
int)+1,abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3 from multi_slot;"
order_qt_select_star "select * from multi_slot order by cast(v['k1'] as
int);"
- // TODO fix and remove enable_rewrite_element_at_to_slot
- order_qt_select_star "select
/*+SET_VAR(enable_rewrite_element_at_to_slot=false) */ abs(cast(v['k4']['k44']
as int)), sum(abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3) from
multi_slot group by abs(cast(v['k4']['k44'] as int))"
+ order_qt_select_star "select abs(cast(v['k4']['k44'] as int)),
sum(abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3) from multi_slot group
by abs(cast(v['k4']['k44'] as int))"
def retry_times = 60
for (def i = 0; i < retry_times; ++i) {
@@ -90,4 +89,31 @@ suite ("multi_slot") {
}
order_qt_select_mv "select abs(cast(v['k1'] as int))+cast(v['k2'] as
int)+1,abs(cast(v['k2'] as int)+2)+cast(v['k3'] as int)+3 from multi_slot order
by abs(cast(v['k1'] as int))+cast(v['k2'] as int)+1,abs(cast(v['k2'] as
int)+2)+cast(v['k3'] as int)+3;"
+ sql "drop table if exists test_mv"
+ sql """
+ CREATE TABLE `test_mv` (
+ `handle_time` datetime NOT NULL ,
+ `client_request` variant NULL,
+ `status` int NULL
+ )
+ DISTRIBUTED BY HASH(`handle_time`)
+ BUCKETS 10 PROPERTIES (
+ "is_being_synced" = "false",
+ "storage_medium" = "hdd",
+ "storage_format" = "V2",
+ "inverted_index_storage_format" = "V1",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false",
+ "replication_num" = "1"
+ );
+ """
+
+ sql """insert into test_mv values ('2021-01-01 11:11:11', '{"url" :
"http://xxx.xxx.xxx"}', 12)"""
+ createMV("create materialized view mv_1 as select `handle_time`,
`client_request`['url'] as `uri`, `status` from test_mv")
+ qt_sql "select `handle_time`, `client_request`['url'] as `uri`, `status`
from test_mv"
+ test {
+ sql "create materialized view mv_x as select `client_request`['url']
as `uri`, `status` from test_mv"
+ exception("The first column could not be float, double or complex type
like array, struct, map, json, variant.")
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
