This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3f4ca3da32 [Bug](CURRENT_TIMESTAMP) Fix wrong default value after
schema change (#16364)
3f4ca3da32 is described below
commit 3f4ca3da32a9d4a5506addbb2304774ea85b08f4
Author: Gabriel <[email protected]>
AuthorDate: Fri Feb 3 17:06:24 2023 +0800
[Bug](CURRENT_TIMESTAMP) Fix wrong default value after schema change
(#16364)
* [Bug](CURRENT_TIMESTAMP) Fix wrong default value after schema change
* update
* update
---
.../java/org/apache/doris/analysis/ColumnDef.java | 20 ++++++--
.../main/java/org/apache/doris/catalog/Column.java | 32 +++++++++++--
.../doris/datasource/HMSExternalCatalog.java | 4 +-
.../datasource/iceberg/IcebergExternalCatalog.java | 2 +-
.../org/apache/doris/external/jdbc/JdbcClient.java | 4 +-
.../date/test_default_current_timestamp.out | 4 ++
.../date/test_default_current_timestamp.groovy | 54 ++++++++++++++++++++++
7 files changed, 107 insertions(+), 13 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
index 49052059d9..123e6e0bbc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ColumnDef.java
@@ -28,11 +28,14 @@ import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeNameFormat;
+import org.apache.doris.common.util.TimeUtils;
import com.google.common.base.Preconditions;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.time.LocalDateTime;
+
// Column definition which is generated by SQL syntax parser
// Syntax:
// name type [key] [agg_type] [NULL | NOT NULL] [DEFAULT default_value]
[comment]
@@ -95,6 +98,16 @@ public class ColumnDef {
public static DefaultValue HLL_EMPTY_DEFAULT_VALUE = new
DefaultValue(true, ZERO);
// default "value", "0" means empty bitmap
public static DefaultValue BITMAP_EMPTY_DEFAULT_VALUE = new
DefaultValue(true, ZERO);
+
+ public boolean isCurrentTimeStamp() {
+ return "CURRENT_TIMESTAMP".equals(value) &&
NOW.equals(defaultValueExprDef.getExprName());
+ }
+
+ public String getValue() {
+ return isCurrentTimeStamp()
+ ?
LocalDateTime.now(TimeUtils.getTimeZone().toZoneId()).toString().replace('T', '
')
+ : value;
+ }
}
// parameter initialized in constructor
@@ -362,12 +375,12 @@ public class ColumnDef {
break;
case DATE:
case DATEV2:
- new DateLiteral(defaultValue,
ScalarType.getDefaultDateType(type));
+ new DateLiteral(defaultValue, scalarType);
break;
case DATETIME:
case DATETIMEV2:
if (defaultValueExprDef == null) {
- new DateLiteral(defaultValue,
ScalarType.getDefaultDateType(type));
+ new DateLiteral(defaultValue, scalarType);
} else {
if
(defaultValueExprDef.getExprName().equals(DefaultValue.NOW)) {
break;
@@ -424,7 +437,8 @@ public class ColumnDef {
public Column toColumn() {
return new Column(name, typeDef.getType(), isKey, aggregateType,
isAllowNull, defaultValue.value, comment,
- visible, defaultValue.defaultValueExprDef,
Column.COLUMN_UNIQUE_ID_INIT_VALUE);
+ visible, defaultValue.defaultValueExprDef,
Column.COLUMN_UNIQUE_ID_INIT_VALUE,
+ defaultValue.getValue());
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
index d1230e6cb1..9a48960ce5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java
@@ -90,6 +90,16 @@ public class Column implements Writable, GsonPostProcessable
{
private ColumnStats stats; // cardinality and selectivity etc.
@SerializedName(value = "children")
private List<Column> children;
+ /**
+ * This is similar as `defaultValue`. Differences are:
+ * 1. `realDefaultValue` indicates the **default underlying literal**.
+ * 2. Instead, `defaultValue` indicates the **original expression** which
is specified by users.
+ *
+ * For example, if user create a table with (columnA, DATETIME, DEFAULT
CURRENT_TIMESTAMP)
+ * `realDefaultValue` here is current date time while `defaultValue` is
`CURRENT_TIMESTAMP`.
+ */
+ @SerializedName(value = "realDefaultValue")
+ private String realDefaultValue;
// Define expr may exist in two forms, one is analyzed, and the other is
not analyzed.
// Currently, analyzed define expr is only used when creating materialized
views,
// so the define expr in RollupJob must be analyzed.
@@ -137,12 +147,18 @@ public class Column implements Writable,
GsonPostProcessable {
public Column(String name, Type type, boolean isKey, AggregateType
aggregateType, boolean isAllowNull,
String defaultValue, String comment) {
this(name, type, isKey, aggregateType, isAllowNull, defaultValue,
comment, true, null,
- COLUMN_UNIQUE_ID_INIT_VALUE);
+ COLUMN_UNIQUE_ID_INIT_VALUE, defaultValue);
+ }
+
+ public Column(String name, Type type, boolean isKey, AggregateType
aggregateType, boolean isAllowNull,
+ String comment, boolean visible, int colUniqueId) {
+ this(name, type, isKey, aggregateType, isAllowNull, null, comment,
visible, null,
+ colUniqueId, null);
}
public Column(String name, Type type, boolean isKey, AggregateType
aggregateType, boolean isAllowNull,
String defaultValue, String comment, boolean visible,
DefaultValueExprDef defaultValueExprDef,
- int colUniqueId) {
+ int colUniqueId, String realDefaultValue) {
this.name = name;
if (this.name == null) {
this.name = "";
@@ -158,6 +174,7 @@ public class Column implements Writable,
GsonPostProcessable {
this.isKey = isKey;
this.isAllowNull = isAllowNull;
this.defaultValue = defaultValue;
+ this.realDefaultValue = realDefaultValue;
this.defaultValueExprDef = defaultValueExprDef;
this.comment = comment;
this.stats = new ColumnStats();
@@ -176,6 +193,7 @@ public class Column implements Writable,
GsonPostProcessable {
this.isCompoundKey = column.isCompoundKey();
this.isAllowNull = column.isAllowNull();
this.defaultValue = column.getDefaultValue();
+ this.realDefaultValue = column.realDefaultValue;
this.defaultValueExprDef = column.defaultValueExprDef;
this.comment = column.getComment();
this.stats = column.getStats();
@@ -393,7 +411,8 @@ public class Column implements Writable,
GsonPostProcessable {
}
tColumn.setIsKey(this.isKey);
tColumn.setIsAllowNull(this.isAllowNull);
- tColumn.setDefaultValue(this.defaultValue);
+ // keep compatibility
+ tColumn.setDefaultValue(this.realDefaultValue == null ?
this.defaultValue : this.realDefaultValue);
tColumn.setVisible(visible);
toChildrenThrift(this, tColumn);
@@ -591,7 +610,8 @@ public class Column implements Writable,
GsonPostProcessable {
@Override
public int hashCode() {
return Objects.hash(name, getDataType(), getStrLen(), getPrecision(),
getScale(), aggregationType,
- isAggregationTypeImplicit, isKey, isAllowNull, defaultValue,
comment, children, visible);
+ isAggregationTypeImplicit, isKey, isAllowNull, defaultValue,
comment, children, visible,
+ realDefaultValue);
}
@Override
@@ -617,7 +637,8 @@ public class Column implements Writable,
GsonPostProcessable {
&& getScale() == other.getScale()
&& Objects.equals(comment, other.comment)
&& visible == other.visible
- && Objects.equals(children, other.children);
+ && Objects.equals(children, other.children)
+ && Objects.equals(realDefaultValue, other.realDefaultValue);
}
@Override
@@ -640,6 +661,7 @@ public class Column implements Writable,
GsonPostProcessable {
notNull = in.readBoolean();
if (notNull) {
defaultValue = Text.readString(in);
+ realDefaultValue = defaultValue;
}
stats = ColumnStats.read(in);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
index 336d31df7e..ccf05e597b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
@@ -192,7 +192,7 @@ public class HMSExternalCatalog extends ExternalCatalog {
for (FieldSchema field : schema) {
tmpSchema.add(new Column(field.getName(),
HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()), true, null,
- true, null, field.getComment(), true, null, -1));
+ true, field.getComment(), true, -1));
}
return tmpSchema;
}
@@ -204,7 +204,7 @@ public class HMSExternalCatalog extends ExternalCatalog {
for (FieldSchema field : hmsSchema) {
tmpSchema.add(new Column(field.getName(),
HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()), true, null,
- true, null, field.getComment(), true, null,
+ true, field.getComment(), true,
schema.caseInsensitiveFindField(field.getName()).fieldId()));
}
return tmpSchema;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
index 53e5b57459..c396ca268f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java
@@ -185,7 +185,7 @@ public abstract class IcebergExternalCatalog extends
ExternalCatalog {
for (Types.NestedField field : columns) {
tmpSchema.add(new Column(field.name(),
icebergTypeToDorisType(field.type()), true, null,
- true, null, field.doc(), true, null, -1));
+ true, field.doc(), true, -1));
}
return tmpSchema;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java
b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java
index 3605fefdd9..8e9d754b8c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/jdbc/JdbcClient.java
@@ -678,8 +678,8 @@ public class JdbcClient {
for (JdbcFieldSchema field : jdbcTableSchema) {
dorisTableSchema.add(new Column(field.getColumnName(),
jdbcTypeToDoris(field), true, null,
- true, null, field.getRemarks(),
- true, null, -1));
+ true, field.getRemarks(),
+ true, -1));
}
return dorisTableSchema;
}
diff --git
a/regression-test/data/datatype_p0/date/test_default_current_timestamp.out
b/regression-test/data/datatype_p0/date/test_default_current_timestamp.out
new file mode 100644
index 0000000000..4ee136aef2
--- /dev/null
+++ b/regression-test/data/datatype_p0/date/test_default_current_timestamp.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+2
+
diff --git
a/regression-test/suites/datatype_p0/date/test_default_current_timestamp.groovy
b/regression-test/suites/datatype_p0/date/test_default_current_timestamp.groovy
new file mode 100644
index 0000000000..1cc6ad8044
--- /dev/null
+++
b/regression-test/suites/datatype_p0/date/test_default_current_timestamp.groovy
@@ -0,0 +1,54 @@
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_default_current_timestamp") {
+ def tbName = "test_default_current_timestamp"
+ sql "DROP TABLE IF EXISTS ${tbName}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbName} (
+ `house_id` bigint(20) NULL DEFAULT "-1" COMMENT '仓库id',
+ `pick_order_big_num` decimal(27, 9) NULL
+ )
+ UNIQUE KEY(house_id)
+ DISTRIBUTED BY HASH(house_id) BUCKETS 5
properties("replication_num" = "1");
+ """
+ sql "insert into ${tbName} values(1,1.1)"
+ sql "insert into ${tbName} values(2,1.1)"
+
+ sql """ ALTER TABLE ${tbName} ADD COLUMN compute_time datetime NULL
DEFAULT CURRENT_TIMESTAMP COMMENT '计算时间' AFTER pick_order_big_num; """
+
+ def getJobState = { tableName ->
+ def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE
IndexName='${tableName}' ORDER BY createtime DESC LIMIT 1 """
+ return jobStateResult[0][9]
+ }
+ int max_try_time = 1000
+ while(max_try_time--){
+ String result = getJobState(tbName)
+ if (result == "FINISHED") {
+ break
+ } else {
+ sleep(100)
+ if (max_try_time < 1){
+ assertEquals(1,2)
+ }
+ }
+ }
+ sql """sync"""
+ qt_sql """ SELECT COUNT(*) FROM ${tbName} WHERE date(compute_time) =
curdate() """
+ sql "DROP TABLE ${tbName}"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
