This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 6d50c28bd2b [fix](schema-change) fix the bug of alter column nullable
when double writing (#41737) (#42351)
6d50c28bd2b is described below
commit 6d50c28bd2b7bbb5b7e082153e4f3916652a30b9
Author: Luwei <[email protected]>
AuthorDate: Wed Oct 30 10:03:28 2024 +0800
[fix](schema-change) fix the bug of alter column nullable when double
writing (#41737) (#42351)
pick master #41737
---
be/src/exec/tablet_info.cpp | 30 ++++++----
.../schema_change_p0/test_alter_uniq_null.groovy | 68 ++++++++++++++++++++++
2 files changed, 88 insertions(+), 10 deletions(-)
diff --git a/be/src/exec/tablet_info.cpp b/be/src/exec/tablet_info.cpp
index 1734a00ad6a..32012b83b7a 100644
--- a/be/src/exec/tablet_info.cpp
+++ b/be/src/exec/tablet_info.cpp
@@ -27,6 +27,7 @@
#include <algorithm>
#include <ostream>
+#include <string>
#include "olap/tablet_schema.h"
#include "runtime/define_primitive_type.h"
@@ -74,7 +75,8 @@ Status OlapTableSchemaParam::init(const
POlapTableSchemaParam& pschema) {
for (auto& col : pschema.partial_update_input_columns()) {
_partial_update_input_columns.insert(col);
}
- std::unordered_map<std::pair<std::string, FieldType>, SlotDescriptor*>
slots_map;
+ std::unordered_map<std::string, SlotDescriptor*> slots_map;
+
_tuple_desc = _obj_pool.add(new TupleDescriptor(pschema.tuple_desc()));
// When FE version is less than 2.0.3, But BE upgrade to 2.0.3,
// the filed col_type in slot_desc is INVALID_TYPE default.
@@ -86,8 +88,10 @@ Status OlapTableSchemaParam::init(const
POlapTableSchemaParam& pschema) {
_tuple_desc->add_slot(slot_desc);
string data_type;
EnumToString(TPrimitiveType, to_thrift(slot_desc->col_type()),
data_type);
- slots_map.emplace(std::make_pair(to_lower(slot_desc->col_name()),
-
TabletColumn::get_field_type_by_string(data_type)),
+ std::string is_null_str = slot_desc->is_nullable() ? "true" : "false";
+ std::string data_type_str =
+
std::to_string(int64_t(TabletColumn::get_field_type_by_string(data_type)));
+ slots_map.emplace(to_lower(slot_desc->col_name()) + "+" +
data_type_str + is_null_str,
slot_desc);
}
@@ -98,10 +102,12 @@ Status OlapTableSchemaParam::init(const
POlapTableSchemaParam& pschema) {
for (auto& pcolumn_desc : p_index.columns_desc()) {
if (!_is_partial_update ||
_partial_update_input_columns.count(pcolumn_desc.name()) > 0) {
- auto it = slots_map.find(std::make_pair(
- to_lower(pcolumn_desc.name()),
- TabletColumn::get_field_type_by_string(
+ std::string is_null_str = pcolumn_desc.is_nullable() ? "true"
: "false";
+ std::string data_type_str =
+
std::to_string(int64_t(TabletColumn::get_field_type_by_string(
has_invalid_type ? "INVALID_TYPE" :
pcolumn_desc.type())));
+ auto it = slots_map.find(to_lower(pcolumn_desc.name()) + "+" +
data_type_str +
+ is_null_str);
if (it == std::end(slots_map)) {
return Status::InternalError("unknown index column,
column={}, type={}",
pcolumn_desc.name(),
pcolumn_desc.type());
@@ -140,7 +146,7 @@ Status OlapTableSchemaParam::init(const
TOlapTableSchemaParam& tschema) {
for (auto& tcolumn : tschema.partial_update_input_columns) {
_partial_update_input_columns.insert(tcolumn);
}
- std::unordered_map<std::pair<std::string, PrimitiveType>, SlotDescriptor*>
slots_map;
+ std::unordered_map<std::string, SlotDescriptor*> slots_map;
_tuple_desc = _obj_pool.add(new TupleDescriptor(tschema.tuple_desc));
// When FE version is less than 2.0.3, But BE upgrade to 2.0.3,
// the filed col_type in slot_desc is INVALID_TYPE default.
@@ -150,7 +156,9 @@ Status OlapTableSchemaParam::init(const
TOlapTableSchemaParam& tschema) {
auto slot_desc = _obj_pool.add(new SlotDescriptor(t_slot_desc));
if (slot_desc->col_type() == INVALID_TYPE) has_invalid_type = true;
_tuple_desc->add_slot(slot_desc);
- slots_map.emplace(std::make_pair(to_lower(slot_desc->col_name()),
slot_desc->col_type()),
+ std::string is_null_str = slot_desc->is_nullable() ? "true" : "false";
+ std::string data_type_str =
std::to_string(int64_t(slot_desc->col_type()));
+ slots_map.emplace(to_lower(slot_desc->col_name()) + "+" +
data_type_str + is_null_str,
slot_desc);
}
@@ -162,10 +170,12 @@ Status OlapTableSchemaParam::init(const
TOlapTableSchemaParam& tschema) {
for (auto& tcolumn_desc : t_index.columns_desc) {
if (!_is_partial_update ||
_partial_update_input_columns.count(tcolumn_desc.column_name)
> 0) {
+ std::string is_null_str = tcolumn_desc.is_allow_null ? "true"
: "false";
TPrimitiveType::type col_type = has_invalid_type ?
TPrimitiveType::INVALID_TYPE
:
tcolumn_desc.column_type.type;
- auto it =
slots_map.find(std::make_pair(to_lower(tcolumn_desc.column_name),
-
thrift_to_type(col_type)));
+ std::string data_type_str =
std::to_string(int64_t(thrift_to_type(col_type)));
+ auto it = slots_map.find(to_lower(tcolumn_desc.column_name) +
"+" + data_type_str +
+ is_null_str);
if (it == slots_map.end()) {
return Status::InternalError("unknown index column,
column={}, type={}",
tcolumn_desc.column_name,
diff --git
a/regression-test/suites/schema_change_p0/test_alter_uniq_null.groovy
b/regression-test/suites/schema_change_p0/test_alter_uniq_null.groovy
new file mode 100644
index 00000000000..47ebbe01983
--- /dev/null
+++ b/regression-test/suites/schema_change_p0/test_alter_uniq_null.groovy
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// The cases is copied from https://github.com/trinodb/trino/tree/master
+// /testing/trino-product-tests/src/main/resources/sql-tests/testcases
+// and modified by Doris.
+
+suite("test_alter_uniq_null") {
+ def tableName = "test_alter_uniq_null_tbl"
+
+ def getJobState = { tableName1 ->
+ def jobStateResult = sql """ SHOW ALTER TABLE COLUMN WHERE
IndexName='${tableName1}' ORDER BY createtime DESC LIMIT 1 """
+ println jobStateResult
+ return jobStateResult[0][9]
+ }
+
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+
+ sql """
+ CREATE TABLE ${tableName} (
+ `k1` VARCHAR(30) NOT NULL,
+ `k2` VARCHAR(24) NOT NULL,
+ `v1` VARCHAR(6) NULL,
+ `v2` INT NOT NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(`k1`, `k2`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`k2`) BUCKETS 1
+ PROPERTIES (
+ "light_schema_change" = "true",
+ "enable_unique_key_merge_on_write" = "false",
+ "replication_num" = "1"
+ );
+ """
+
+ sql """alter table ${tableName} modify column `v2` INT NULL"""
+ sleep(10)
+ max_try_num = 1000
+ while (max_try_num--) {
+ String res = getJobState(tableName)
+ if (res == "FINISHED" || res == "CANCELLED") {
+ assertEquals("FINISHED", res)
+ break
+ } else {
+ int val = 100000 + max_try_num
+ sql """ insert into ${tableName} values ("${val}", "client", "3",
100), ("${val}", "client", "4", 200)"""
+ sleep(10)
+ if (max_try_num < 1) {
+ println "test timeout," + "state:" + res
+ assertEquals("FINISHED",res)
+ }
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
