This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new def0ed37f84 [improve](schema-change) support nested type with varchar
type to support length growing (#46639)
def0ed37f84 is described below
commit def0ed37f844d9f48150ed813c6f76fd0d7a9fff
Author: amory <[email protected]>
AuthorDate: Wed Jan 15 11:13:54 2025 +0800
[improve](schema-change) support nested type with varchar type to support
length growing (#46639)
The array|map|struct schema-change behavior supports moditfy to change
the length of the varchar type in the current column type.
before
```
mysql> alter table t_sc MODIFY COLUMN s struct<col:varchar(20)>;
ERROR 1105 (HY000): errCode = 2, detailMessage = Can not change
struct<col:varchar(10)> to `__doris_shadow_s` struct<col:varchar(20)> NULL
```
---
.../apache/doris/alter/SchemaChangeHandler.java | 8 +-
.../main/java/org/apache/doris/catalog/Column.java | 15 +-
.../java/org/apache/doris/catalog/ColumnType.java | 55 ++++++
.../test_varchar_sc_in_complex.out | 34 ++++
.../test_varchar_sc_in_complex.groovy | 187 +++++++++++++++++++++
5 files changed, 286 insertions(+), 13 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
index af2a0a7a9bd..45064092069 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeHandler.java
@@ -38,6 +38,7 @@ import org.apache.doris.analysis.SlotRef;
import org.apache.doris.catalog.AggregateType;
import org.apache.doris.catalog.BinlogConfig;
import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.ColumnType;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DistributionInfo;
import org.apache.doris.catalog.DistributionInfo.DistributionInfoType;
@@ -629,7 +630,12 @@ public class SchemaChangeHandler extends AlterHandler {
// TODO:the case where columnPos is not empty has not been
considered
if (columnPos == null && col.getDataType() ==
PrimitiveType.VARCHAR
&& modColumn.getDataType() ==
PrimitiveType.VARCHAR) {
- col.checkSchemaChangeAllowed(modColumn);
+
ColumnType.checkSupportSchemaChangeForCharType(col.getType(),
modColumn.getType());
+ lightSchemaChange =
olapTable.getEnableLightSchemaChange();
+ }
+ if (columnPos == null && col.getDataType().isComplexType()
+ && modColumn.getDataType().isComplexType()) {
+
ColumnType.checkSupportSchemaChangeForComplexType(col.getType(),
modColumn.getType(), true);
lightSchemaChange =
olapTable.getEnableLightSchemaChange();
}
if (col.isClusterKey()) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index 5a87563bd18..fa7c7feb7f0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -847,11 +847,6 @@ public class Column implements GsonPostProcessable {
throw new DdlException("Dest column name is empty");
}
- // now nested type can only support change order
- if (type.isComplexType() && !type.equals(other.type)) {
- throw new DdlException("Can not change " + type + " to " + other);
- }
-
if (!ColumnType.isSchemaChangeAllowed(type, other.type)) {
throw new DdlException("Can not change " + getDataType() + " to "
+ other.getDataType());
}
@@ -888,13 +883,9 @@ public class Column implements GsonPostProcessable {
}
}
- if ((getDataType() == PrimitiveType.VARCHAR && other.getDataType() ==
PrimitiveType.VARCHAR) || (
- getDataType() == PrimitiveType.CHAR && other.getDataType() ==
PrimitiveType.VARCHAR) || (
- getDataType() == PrimitiveType.CHAR && other.getDataType() ==
PrimitiveType.CHAR)) {
- if (getStrLen() > other.getStrLen()) {
- throw new DdlException("Cannot shorten string length");
- }
- }
+ // Nested types only support changing the order and increasing the
length of the nested char type
+ // Char-type only support length growing
+ ColumnType.checkSupportSchemaChangeForComplexType(type, other.type,
false);
// now we support convert decimal to varchar type
if ((getDataType() == PrimitiveType.DECIMALV2 ||
getDataType().isDecimalV3Type())
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
index 458a6a46110..234c0d4eef0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/ColumnType.java
@@ -17,6 +17,7 @@
package org.apache.doris.catalog;
+import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.io.Text;
import org.apache.doris.persist.gson.GsonUtils;
@@ -168,6 +169,60 @@ public abstract class ColumnType {
return
schemaChangeMatrix[lhs.getPrimitiveType().ordinal()][rhs.getPrimitiveType().ordinal()];
}
+ // This method defines the char type
+ // to support the schema-change behavior of length growth.
+ // return true if the checkType and other are both char-type otherwise
return false,
+ // which used in checkSupportSchemaChangeForComplexType
+ public static boolean checkSupportSchemaChangeForCharType(Type checkType,
Type other) throws DdlException {
+ if ((checkType.getPrimitiveType() == PrimitiveType.VARCHAR &&
other.getPrimitiveType() == PrimitiveType.VARCHAR)
+ || (checkType.getPrimitiveType() == PrimitiveType.CHAR
+ && other.getPrimitiveType() == PrimitiveType.VARCHAR)
+ || (checkType.getPrimitiveType() == PrimitiveType.CHAR
+ && other.getPrimitiveType() == PrimitiveType.CHAR)) {
+ if (checkType.getLength() > other.getLength()) {
+ throw new DdlException("Cannot shorten string length");
+ }
+ return true;
+ } else {
+ // types equal can return true
+ return checkType.equals(other);
+ }
+ }
+
+ // This method defines the complex type which is struct, array, map if
nested char-type
+ // to support the schema-change behavior of length growth.
+ public static void checkSupportSchemaChangeForComplexType(Type checkType,
Type other, boolean nested)
+ throws DdlException {
+ if (checkType.isStructType() && other.isStructType()) {
+ StructType thisStructType = (StructType) checkType;
+ StructType otherStructType = (StructType) other;
+ if (thisStructType.getFields().size() !=
otherStructType.getFields().size()) {
+ throw new DdlException("Cannot change struct type with
different field size");
+ }
+ for (int i = 0; i < thisStructType.getFields().size(); i++) {
+
checkSupportSchemaChangeForComplexType(thisStructType.getFields().get(i).getType(),
+ otherStructType.getFields().get(i).getType(), true);
+ }
+ } else if (checkType.isArrayType()) {
+ if (!other.isArrayType()) {
+ throw new DdlException("Cannot change " + checkType.toSql() +
" to " + other.toSql());
+ }
+ checkSupportSchemaChangeForComplexType(((ArrayType)
checkType).getItemType(),
+ ((ArrayType) other).getItemType(), true);
+ } else if (checkType.isMapType() && other.isMapType()) {
+ checkSupportSchemaChangeForComplexType(((MapType)
checkType).getKeyType(),
+ ((MapType) other).getKeyType(), true);
+ checkSupportSchemaChangeForComplexType(((MapType)
checkType).getValueType(),
+ ((MapType) other).getValueType(), true);
+ } else {
+ // only support char-type schema change behavior for nested
complex type
+ // if nested is false, we do not check return value.
+ if (nested && !checkSupportSchemaChangeForCharType(checkType,
other)) {
+ throw new DdlException("Cannot change " + checkType.toSql() +
" to " + other.toSql());
+ }
+ }
+ }
+
public static void write(DataOutput out, Type type) throws IOException {
Preconditions.checkArgument(type.isScalarType() ||
type.isAggStateType()
|| type.isArrayType() || type.isMapType() ||
type.isStructType(),
diff --git
a/regression-test/data/schema_change_p0/test_varchar_sc_in_complex.out
b/regression-test/data/schema_change_p0/test_varchar_sc_in_complex.out
new file mode 100644
index 00000000000..c852a4008fe
--- /dev/null
+++ b/regression-test/data/schema_change_p0/test_varchar_sc_in_complex.out
@@ -0,0 +1,34 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sc_origin --
+0 ["2025-01-01"] {"amory":"better"} {"col":"commiter"}
+1 ["2025-01-02"] {"doris":"better"} {"col":"amory"}
+
+-- !sc_after --
+0 ["2025-01-01"] {"amory":"better"} {"col":"commiter"}
+1 ["2025-01-02"] {"doris":"better"} {"col":"amory"}
+2 ["2025-01-03-22-33"] {"doris":"better"} {"col":"amory"}
+
+-- !sc_origin --
+0 ["2025-01-01"] {"amory":"better"} {"col":"commiter"}
+1 ["2025-01-02"] {"doris":"better"} {"col":"amory"}
+2 ["2025-01-03-22-33"] {"doris":"better"} {"col":"amory"}
+
+-- !sc_after --
+0 ["2025-01-01"] {"amory":"better"} {"col":"commiter"}
+1 ["2025-01-02"] {"doris":"better"} {"col":"amory"}
+2 ["2025-01-03-22-33"] {"doris":"better"} {"col":"amory"}
+3 ["2025-01-03-22-33"] {"doris111111111":"better2222222222"}
{"col":"amory"}
+
+-- !sc_origin --
+0 ["2025-01-01"] {"amory":"better"} {"col":"commiter"}
+1 ["2025-01-02"] {"doris":"better"} {"col":"amory"}
+2 ["2025-01-03-22-33"] {"doris":"better"} {"col":"amory"}
+3 ["2025-01-03-22-33"] {"doris111111111":"better2222222222"}
{"col":"amory"}
+
+-- !sc_after --
+0 ["2025-01-01"] {"amory":"better"} {"col":"commiter"}
+1 ["2025-01-02"] {"doris":"better"} {"col":"amory"}
+2 ["2025-01-03-22-33"] {"doris":"better"} {"col":"amory"}
+3 ["2025-01-03-22-33"] {"doris111111111":"better2222222222"}
{"col":"amory"}
+4 ["2025-01-03-22-33"] {"doris":"better"} {"col":"amoryIsBetter"}
+
diff --git
a/regression-test/suites/schema_change_p0/test_varchar_sc_in_complex.groovy
b/regression-test/suites/schema_change_p0/test_varchar_sc_in_complex.groovy
new file mode 100644
index 00000000000..d3d2554025f
--- /dev/null
+++ b/regression-test/suites/schema_change_p0/test_varchar_sc_in_complex.groovy
@@ -0,0 +1,187 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+import java.util.concurrent.TimeUnit
+import org.awaitility.Awaitility
+
+suite ("test_varchar_sc_in_complex") {
+ def getJobState = { tableName ->
+ def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE
IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """
+ return jobStateResult[0][9]
+ }
+
+ def tableName = "test_varchar_sc_in_complex"
+
+ try {
+
+ String backend_id;
+ def backendId_to_backendIP = [:]
+ def backendId_to_backendHttpPort = [:]
+ getBackendIpHttpPort(backendId_to_backendIP,
backendId_to_backendHttpPort);
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `c0` LARGEINT NOT NULL,
+ `c_a` ARRAY<VARCHAR(10)>,
+ `c_m` MAP<VARCHAR(10),VARCHAR(10)>,
+ `c_s` STRUCT<col:VARCHAR(10)>
+ ) DISTRIBUTED BY HASH(c0) BUCKETS 1
+ PROPERTIES ( "replication_num" = "1", "light_schema_change" =
"true" )
+ """
+
+ sql """ insert into ${tableName} values
+ (0,['2025-01-01'], {'amory':'better'},
named_struct('col','commiter'));
+ """
+ sql """ insert into ${tableName} values
+ (1,['2025-01-02'], {'doris':'better'},
named_struct('col','amory'));
+ """
+ // this can be insert but with cut off the left string to 10
+ test {
+ sql """ insert into ${tableName} values
+ (11, ['2025-01-03-22-33'],
{'doris111111111':'better2222222222'}, named_struct('col','amoryIsBetter'));
+ """
+ exception "Insert has filtered data in strict mode"
+ }
+
+ test {
+ sql """ alter table ${tableName} modify column c_a
array<varchar(3)>
+ """
+ exception "Cannot shorten string length"
+ }
+
+ test {
+ sql """ alter table ${tableName} modify column c_m
map<varchar(3),varchar(3)>
+ """
+ exception "Cannot shorten string length"
+ }
+
+ test {
+ sql """ alter table ${tableName} modify column c_s
struct<col:varchar(3)>
+ """
+ exception "Cannot shorten string length"
+ }
+
+ // add case alter modify array/map/struct to other type
+ // test array to struct
+ test {
+ sql """ alter table ${tableName} modify column c_a
struct<col:varchar(3)>
+ """
+ exception "Can not change ARRAY to STRUCT"
+ }
+ // test array to map
+ test {
+ sql """ alter table ${tableName} modify column c_a
map<varchar(3),varchar(3)>
+ """
+ exception "Can not change ARRAY to MAP"
+ }
+ // test map to array
+ test {
+ sql """ alter table ${tableName} modify column c_m
array<varchar(3)>
+ """
+ exception "Can not change MAP to ARRAY"
+ }
+ // test map to struct
+ test {
+ sql """ alter table ${tableName} modify column c_m
struct<col:varchar(3)>
+ """
+ exception "Can not change MAP to STRUCT"
+ }
+ // test struct to array
+ test {
+ sql """ alter table ${tableName} modify column c_s
array<varchar(3)>
+ """
+ exception "Can not change STRUCT to ARRAY"
+ }
+ // test struct to map
+ test {
+ sql """ alter table ${tableName} modify column c_s
map<varchar(3),varchar(3)>
+ """
+ exception "Can not change STRUCT to MAP"
+ }
+
+
+ sql """ alter table ${tableName} modify column c_a array<varchar(20)>
"""
+ int max_try_secs = 300
+ Awaitility.await().atMost(max_try_secs,
TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(()
-> {
+ String result = getJobState(tableName)
+ if (result == "FINISHED") {
+ return true;
+ }
+ return false;
+ });
+
+ String[][] res = sql """ desc ${tableName} """
+ logger.info(res[1][1])
+ // array varchar(10)
+ assertEquals(res[1][1].toLowerCase(),"array<varchar(20)>")
+
+ qt_sc_origin " select * from ${tableName} order by c0; "
+
+ // then insert some data to modified array with varchar(20)
+ sql """ insert into ${tableName} values
+ (2,['2025-01-03-22-33'], {'doris':'better'},
named_struct('col','amory'));
+ """
+ qt_sc_after " select * from ${tableName} order by c0; "
+
+ // map varchar(10)
+ sql """ alter table ${tableName} modify column c_m
map<varchar(20),varchar(20)> """
+ Awaitility.await().atMost(max_try_secs,
TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(()
-> {
+ String result = getJobState(tableName)
+ if (result == "FINISHED") {
+ return true;
+ }
+ return false;
+ });
+
+ res = sql """ desc ${tableName} """
+ logger.info(res[2][1])
+ assertEquals(res[2][1].toLowerCase(),"map<varchar(20),varchar(20)>")
+
+ // insert some data to modified map with varchar(20)
+ qt_sc_origin " select * from ${tableName} order by c0; "
+ sql """ insert into ${tableName} values
+ (3,['2025-01-03-22-33'],
{'doris111111111':'better2222222222'}, named_struct('col','amory'));
+ """
+ qt_sc_after " select * from ${tableName} order by c0; "
+
+ // struct varchar(10)
+ sql """ alter table ${tableName} modify column c_s
struct<col:varchar(20)> """
+ Awaitility.await().atMost(max_try_secs,
TimeUnit.SECONDS).with().pollDelay(100, TimeUnit.MILLISECONDS).await().until(()
-> {
+ String result = getJobState(tableName)
+ if (result == "FINISHED") {
+ return true;
+ }
+ return false;
+ });
+
+ res = sql """ desc ${tableName} """
+ logger.info(res[3][1])
+ assertEquals(res[3][1].toLowerCase(),"struct<col:varchar(20)>")
+
+ // insert some data to modified struct with varchar(20)
+ qt_sc_origin " select * from ${tableName} order by c0; "
+ sql """ insert into ${tableName} values
+ (4,['2025-01-03-22-33'], {'doris':'better'},
named_struct('col','amoryIsBetter'));
+ """
+ qt_sc_after " select * from ${tableName} order by c0; "
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${tableName}")
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
