This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new bc514ef4e6 Revert "[fix](testcase) fix test case failure of insert
null value into not null column (#20963)" (#23501)
bc514ef4e6 is described below
commit bc514ef4e69c15f5515d1bc9bdcbdf828cf68f00
Author: TengJianPing <[email protected]>
AuthorDate: Fri Aug 25 17:48:12 2023 +0800
Revert "[fix](testcase) fix test case failure of insert null value into not
null column (#20963)" (#23501)
This reverts commit 55a6649da962fb170ddb40fea8ef26bdc552a51a.
Mannual Revert "fix in strict mode, return error for insert if datatype
convert fails (#20378)"
This mannual reverts commit 1b94b6368f5e871c9a0fe53dd7c64409079a4c9d
---
be/src/runtime/runtime_state.h | 4 -
be/src/vec/functions/function_cast.h | 76 +++-----
.../java/org/apache/doris/qe/SessionVariable.java | 1 -
regression-test/data/insert_p0/insert_invalid.out | 11 --
.../suites/insert_p0/insert_invalid.groovy | 215 ---------------------
.../query_p0/join/test_full_join_batch_size.groovy | 16 +-
regression-test/suites/statistics/test_ddl.groovy | 1 -
7 files changed, 34 insertions(+), 290 deletions(-)
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index 1f35f6c1b8..0e7c6b7ab3 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -420,10 +420,6 @@ public:
: 0;
}
- bool enable_insert_strict() const {
- return _query_options.__isset.enable_insert_strict &&
_query_options.enable_insert_strict;
- }
-
private:
Status create_error_log_file();
diff --git a/be/src/vec/functions/function_cast.h
b/be/src/vec/functions/function_cast.h
index 93fdd1f108..528ba75a21 100644
--- a/be/src/vec/functions/function_cast.h
+++ b/be/src/vec/functions/function_cast.h
@@ -86,7 +86,6 @@
#include "vec/functions/function_helpers.h"
#include "vec/io/reader_buffer.h"
#include "vec/runtime/vdatetime_value.h"
-#include "vec/utils/template_helpers.hpp"
class DateLUTImpl;
@@ -1357,58 +1356,35 @@ struct ConvertThroughParsing {
offsets = &col_from_string->get_offsets();
}
- bool is_load = (context && context->state()->query_type() ==
TQueryType::type::LOAD);
- bool is_strict_insert = (context &&
context->state()->enable_insert_strict());
size_t current_offset = 0;
- auto status = std::visit(
- [&](auto is_load_, auto is_strict_insert_) {
- for (size_t i = 0; i < size; ++i) {
- size_t next_offset = std::is_same_v<FromDataType,
DataTypeString>
- ? (*offsets)[i]
- : (current_offset +
fixed_string_size);
- size_t string_size = std::is_same_v<FromDataType,
DataTypeString>
- ? next_offset -
current_offset
- : fixed_string_size;
-
- ReadBuffer read_buffer(&(*chars)[current_offset],
string_size);
-
- bool parsed;
- if constexpr (IsDataTypeDecimal<ToDataType>) {
- parsed = try_parse_impl<ToDataType>(
- vec_to[i], read_buffer,
context->state()->timezone_obj(),
- time_zone_cache, vec_to.get_scale());
- } else if constexpr (IsDataTypeDateTimeV2<ToDataType>)
{
- auto type =
check_and_get_data_type<DataTypeDateTimeV2>(
- block.get_by_position(result).type.get());
- parsed = try_parse_impl<ToDataType>(vec_to[i],
read_buffer,
-
context->state()->timezone_obj(),
-
time_zone_cache, type->get_scale());
- } else {
- parsed = try_parse_impl<ToDataType, void*,
FromDataType>(
- vec_to[i], read_buffer,
context->state()->timezone_obj(),
- time_zone_cache);
- }
- (*vec_null_map_to)[i] = !parsed ||
!is_all_read(read_buffer);
- if constexpr (is_load_ && is_strict_insert_) {
- if (string_size != 0 && (*vec_null_map_to)[i]) {
- return Status::InternalError(
- "Invalid value {} in strict mode for
function {}, source "
- "column {}, from "
- "type "
- "{} to type {}",
-
std::string((char*)&(*chars)[current_offset], string_size),
- Name::name, col_from->get_name(),
FromDataType().get_name(),
- ToDataType().get_name());
- }
- }
+ for (size_t i = 0; i < size; ++i) {
+ size_t next_offset = std::is_same_v<FromDataType, DataTypeString>
+ ? (*offsets)[i]
+ : (current_offset +
fixed_string_size);
+ size_t string_size = std::is_same_v<FromDataType, DataTypeString>
+ ? next_offset - current_offset
+ : fixed_string_size;
- current_offset = next_offset;
- }
- return Status::OK();
- },
- make_bool_variant(is_load),
make_bool_variant(is_strict_insert));
+ ReadBuffer read_buffer(&(*chars)[current_offset], string_size);
- RETURN_IF_ERROR(status);
+ bool parsed;
+ if constexpr (IsDataTypeDecimal<ToDataType>) {
+ parsed = try_parse_impl<ToDataType>(vec_to[i], read_buffer,
+
context->state()->timezone_obj(),
+ time_zone_cache,
vec_to.get_scale());
+ } else if constexpr (IsDataTypeDateTimeV2<ToDataType>) {
+ auto type = check_and_get_data_type<DataTypeDateTimeV2>(
+ block.get_by_position(result).type.get());
+ parsed = try_parse_impl<ToDataType>(vec_to[i], read_buffer,
+
context->state()->timezone_obj(),
+ time_zone_cache,
type->get_scale());
+ } else {
+ parsed = try_parse_impl<ToDataType, void*, FromDataType>(
+ vec_to[i], read_buffer,
context->state()->timezone_obj(), time_zone_cache);
+ }
+ (*vec_null_map_to)[i] = !parsed || !is_all_read(read_buffer);
+ current_offset = next_offset;
+ }
block.get_by_position(result).column =
ColumnNullable::create(std::move(col_to),
std::move(col_null_map_to));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 3e26c8fab3..4ed8084d33 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -2151,7 +2151,6 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setEnableParquetLazyMat(enableParquetLazyMat);
tResult.setEnableOrcLazyMat(enableOrcLazyMat);
- tResult.setEnableInsertStrict(enableInsertStrict);
tResult.setTruncateCharOrVarcharColumns(truncateCharOrVarcharColumns);
return tResult;
diff --git a/regression-test/data/insert_p0/insert_invalid.out
b/regression-test/data/insert_p0/insert_invalid.out
deleted file mode 100644
index 7e268e6c7e..0000000000
--- a/regression-test/data/insert_p0/insert_invalid.out
+++ /dev/null
@@ -1,11 +0,0 @@
--- This file is automatically generated. You should know what you did if you
want to edit this
--- !select_inserted0 --
-12345678908876643 a
-1234567890887664643 b
-
--- !select_inserted1 --
-12345678908876643 a
-1234567890887664643 b
-
--- !select_inserted2 --
-
diff --git a/regression-test/suites/insert_p0/insert_invalid.groovy
b/regression-test/suites/insert_p0/insert_invalid.groovy
deleted file mode 100644
index 6b3e99ab35..0000000000
--- a/regression-test/suites/insert_p0/insert_invalid.groovy
+++ /dev/null
@@ -1,215 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// The cases is copied from https://github.com/trinodb/trino/tree/master
-// /testing/trino-product-tests/src/main/resources/sql-tests/testcases
-// and modified by Doris.
-suite("insert_invalid") {
- sql """ DROP TABLE IF EXISTS datatype_invalid; """
- sql """
- CREATE TABLE `datatype_invalid` (`timea` bigint NOT NULL, `creatr`
varchar(30) NULL)
- UNIQUE KEY(`timea`)
- DISTRIBUTED BY HASH(`timea`) BUCKETS 1
- PROPERTIES ("replication_num" = "1");
- """
-
- // strict insert
- sql """ set enable_insert_strict=true; """
-
- // test insert select: out of range value
- sql """ DROP TABLE IF EXISTS datatype_invalid_base; """
- sql """
- CREATE TABLE `datatype_invalid_base` (
- `timea` varchar(30) NULL,
- `creatr` varchar(30) NULL
- ) UNIQUE KEY(`timea`)
- DISTRIBUTED BY HASH(`timea`) BUCKETS 1
- PROPERTIES ("replication_num" = "1");
- """
-
- sql """
- insert into
- datatype_invalid_base
- values
- ("12345678908876643", "a"),
- ("1234567890887664643", "b"),
- ("123456789088766445456", "c");
- """
-
- test {
- sql """ insert into datatype_invalid select * from
datatype_invalid_base;"""
- exception "Invalid value"
- }
-
- // test insert select: invalid value
- sql """ DROP TABLE IF EXISTS datatype_invalid_base; """
- sql """
- CREATE TABLE `datatype_invalid_base` (
- `timea` varchar(30) NULL,
- `creatr` varchar(30) NULL
- ) UNIQUE KEY(`timea`)
- DISTRIBUTED BY HASH(`timea`) BUCKETS 1
- PROPERTIES ("replication_num" = "1");
- """
-
- sql """
- insert into
- datatype_invalid_base
- values
- ("a", "a");
- """
- test {
- sql """ insert into datatype_invalid select * from
datatype_invalid_base;"""
- exception "Invalid value"
- }
-
- // test insert select: invalid value
- sql """ DROP TABLE IF EXISTS datatype_invalid_base; """
- sql """
- CREATE TABLE `datatype_invalid_base` (
- `timea` varchar(30) NULL,
- `creatr` varchar(30) NULL
- ) UNIQUE KEY(`timea`)
- DISTRIBUTED BY HASH(`timea`) BUCKETS 1
- PROPERTIES ("replication_num" = "1");
- """
-
- sql """
- insert into
- datatype_invalid_base
- values
- (" ", "a");
- """
- test {
- sql """ insert into datatype_invalid select * from
datatype_invalid_base;"""
- exception "Invalid value"
- }
-
- // test insert select: null into not nullable
- sql """ DROP TABLE IF EXISTS datatype_invalid_base; """
- sql """
- CREATE TABLE `datatype_invalid_base` (
- `timea` varchar(30) NULL,
- `creatr` varchar(30) NULL
- ) UNIQUE KEY(`timea`)
- DISTRIBUTED BY HASH(`timea`) BUCKETS 1
- PROPERTIES ("replication_num" = "1");
- """
-
- sql """
- insert into
- datatype_invalid_base
- values
- (null, "a");
- """
- test {
- sql """ insert into datatype_invalid select * from
datatype_invalid_base;"""
- exception "Insert has filtered data in strict mode"
- }
-
- // test insert
- test {
- sql """ insert into datatype_invalid values("a", "a");"""
- exception "Invalid number format"
- }
- test {
- sql """ insert into datatype_invalid values(" ", "a");"""
- exception "Invalid number format"
- }
- test {
- sql """ insert into datatype_invalid values(123456789088766445456,
"a");"""
- exception "Number out of range"
- }
- test {
- sql """ insert into datatype_invalid values(null, "a");"""
- exception "Insert has filtered data in strict mode"
- }
-
- sql """ DROP TABLE IF EXISTS datatype_invalid; """
- sql """
- CREATE TABLE `datatype_invalid` (`timea` datetime NOT NULL, `creatr`
varchar(30) NULL)
- UNIQUE KEY(`timea`)
- DISTRIBUTED BY HASH(`timea`) BUCKETS 1
- PROPERTIES ("replication_num" = "1");
- """
- test {
- sql """ insert into datatype_invalid values ('2022-02-29', 'a'); """
- exception "Invalid value"
- }
-
- sql """ set enable_insert_strict=false; """
- sql """ DROP TABLE IF EXISTS datatype_invalid; """
- sql """
- CREATE TABLE `datatype_invalid` (`timea` bigint NOT NULL, `creatr`
varchar(30) NULL)
- UNIQUE KEY(`timea`)
- DISTRIBUTED BY HASH(`timea`) BUCKETS 1
- PROPERTIES ("replication_num" = "1");
- """
-
- // non strict insert into select
- sql """ DROP TABLE IF EXISTS datatype_invalid_base; """
- sql """
- CREATE TABLE `datatype_invalid_base` (
- `timea` varchar(30) NULL,
- `creatr` varchar(30) NULL
- ) UNIQUE KEY(`timea`)
- DISTRIBUTED BY HASH(`timea`) BUCKETS 1
- PROPERTIES ("replication_num" = "1");
- """
- sql """
- insert into
- datatype_invalid_base
- values
- ("a", "a"),
- (" ", "a"),
- ("12345678908876643", "a"),
- ("1234567890887664643", "b"),
- ("123456789088766445456", "c");
- """
- sql """ insert into datatype_invalid select * from
datatype_invalid_base;"""
-
- qt_select_inserted0 """ select * from datatype_invalid order by timea """
-
- sql """ DROP TABLE IF EXISTS datatype_invalid_base; """
- sql """
- CREATE TABLE `datatype_invalid_base` (
- `timea` varchar(30) NULL,
- `creatr` varchar(30) NULL
- ) UNIQUE KEY(`timea`)
- DISTRIBUTED BY HASH(`timea`) BUCKETS 1
- PROPERTIES ("replication_num" = "1");
- """
- sql """
- insert into
- datatype_invalid_base
- values
- (null, "a");
- """
- sql """ insert into datatype_invalid select * from
datatype_invalid_base;"""
-
- qt_select_inserted1 """ select * from datatype_invalid order by timea """
-
- sql """ DROP TABLE IF EXISTS datatype_invalid; """
- sql """
- CREATE TABLE `datatype_invalid` (`timea` datetime NOT NULL, `creatr`
varchar(30) NULL)
- UNIQUE KEY(`timea`)
- DISTRIBUTED BY HASH(`timea`) BUCKETS 1
- PROPERTIES ("replication_num" = "1");
- """
- sql """ insert into datatype_invalid values ('2022-02-29', 'a'); """
- qt_select_inserted2 """ select * from datatype_invalid order by timea """
-}
diff --git
a/regression-test/suites/query_p0/join/test_full_join_batch_size.groovy
b/regression-test/suites/query_p0/join/test_full_join_batch_size.groovy
index 6ea3923fa2..7e7a70a884 100644
--- a/regression-test/suites/query_p0/join/test_full_join_batch_size.groovy
+++ b/regression-test/suites/query_p0/join/test_full_join_batch_size.groovy
@@ -16,25 +16,25 @@
// under the License.
suite("test_full_join_batch_size", "query,p0") {
- sql " drop table if exists test_left_join_batch_size_l; ";
- sql " drop table if exists test_left_join_batch_size_r; ";
+ sql " drop table if exists test_full_join_batch_size_l; ";
+ sql " drop table if exists test_full_join_batch_size_r; ";
sql """
- create table test_left_join_batch_size_l (
+ create table test_full_join_batch_size_l (
k1 int,
v1 int
) distributed by hash(k1) buckets 3
properties("replication_num" = "1");
"""
sql """
- create table test_left_join_batch_size_r (
+ create table test_full_join_batch_size_r (
k1 int,
v1 int
) distributed by hash(k1) buckets 3
properties("replication_num" = "1");
"""
- sql """ insert into test_left_join_batch_size_l values (1, 11), (1, 111),
(1, 1111) """
- sql """ insert into test_left_join_batch_size_r values (1, null), (1,
211), (1, 311), (1, 411) """
+ sql """ insert into test_full_join_batch_size_l values (1, 11), (1, 111),
(1, 1111) """
+ sql """ insert into test_full_join_batch_size_r values (1, null), (1,
211), (1, 311), (1, 411) """
qt_sql1 """
select /*+SET_VAR(batch_size=3)*/
@@ -43,8 +43,8 @@ suite("test_full_join_batch_size", "query,p0") {
r.k1,
r.v1
from
- test_left_join_batch_size_l l
- full join test_left_join_batch_size_r r on (
+ test_full_join_batch_size_l l
+ full join test_full_join_batch_size_r r on (
r.v1 = 0
or r.v1 is null
)
diff --git a/regression-test/suites/statistics/test_ddl.groovy
b/regression-test/suites/statistics/test_ddl.groovy
index f5a67d6d55..06e1c72e16 100644
--- a/regression-test/suites/statistics/test_ddl.groovy
+++ b/regression-test/suites/statistics/test_ddl.groovy
@@ -44,7 +44,6 @@ suite("test_ddl") {
INSERT INTO `agg_all_for_analyze_test` VALUES
(-24673,0,-127,-1939606877,-105278987563368327,-257119.385,'bA5rPeM244SovDhOOQ02CmXeM69uhJ8GSHtU','2022-09-28','2022-08-23
01:34:09','wrHimKN3w24QvUiplB9HFWdeCCeX0bQbbFima85zhb1kQ0s6lP6ctie2oGuKF',-4060736.642127,NULL,'22bCQDgO6A0FJB22Q9bASB8cHnYqHeKKGsa1e','-6225805734985728798'),(-22254,0,28,702265972,-6301108547516189202,-667430.114,'M0sReWtDXk7zt7AiDCzuqciSo0JuZzNI3Kez','2022-11-24','2022-11-29
22:52:56','gxsUl9OwrHYuy8Ih0A6XShMYk [...]
"""
- sql "set enable_insert_strict=false"
sql """
INSERT INTO __internal_schema.column_statistics SELECT id,
catalog_id, db_id, tbl_id, idx_id, col_id,
part_id, row_count, ndv, null_count, min, max, data_size,
update_time
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
