This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 8ebeb69bbd7 branch-3.1: [case](tvf) add tvf abnormal cases #56072
(#56130)
8ebeb69bbd7 is described below
commit 8ebeb69bbd768fb61e1f5ba0727da4645d610370
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Sep 25 17:06:14 2025 +0800
branch-3.1: [case](tvf) add tvf abnormal cases #56072 (#56130)
Cherry-picked from #56072
Co-authored-by: MoanasDaddyXu <[email protected]>
---
.../data/load_p0/tvf/test_tvf_error_data.out | 40 ++++
.../load_p0/tvf/test_tvf_error_column.groovy | 160 ++++++++++++++++
.../suites/load_p0/tvf/test_tvf_error_data.groovy | 202 +++++++++++++++++++++
3 files changed, 402 insertions(+)
diff --git a/regression-test/data/load_p0/tvf/test_tvf_error_data.out
b/regression-test/data/load_p0/tvf/test_tvf_error_data.out
new file mode 100644
index 00000000000..af864e1d423
--- /dev/null
+++ b/regression-test/data/load_p0/tvf/test_tvf_error_data.out
@@ -0,0 +1,40 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select --
+1 \N "a" "b" "c" "d"
+
+-- !select --
+1 2025-10-01T10:00:00.001 a b c d
+
+-- !select --
+4 2025-10-01T10:00:00.001 "a"b" "b" "c" "d"
+
+-- !sql --
+1
+
+-- !select --
+3 2025-10-01T10:00:00.001 "a"b"" "b" "c" "d"
+
+-- !sql --
+1
+
+-- !select --
+5 2025-10-01T10:00:00.001 "a\\x" "b\\n" "c\\t" "d\\w\\d"
+
+-- !sql --
+1
+
+-- !select --
+7 2025-10-01T10:00:00.001 "a'b" "b'c'" "c" "d"
+
+-- !sql --
+1
+
+-- !select --
+7 2025-10-01T10:00:00.001 "a'b" "b'c'" "c" "d"
+
+-- !sql --
+1
+
+-- !select --
+8 2025-10-01T10:00:00.001 a,b,c b c d
+
diff --git a/regression-test/suites/load_p0/tvf/test_tvf_error_column.groovy
b/regression-test/suites/load_p0/tvf/test_tvf_error_column.groovy
new file mode 100644
index 00000000000..a0e1184dd36
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/test_tvf_error_column.groovy
@@ -0,0 +1,160 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_tvf_error_column", "p0") {
+ String ak = getS3AK()
+ String sk = getS3SK()
+ String s3_endpoint = getS3Endpoint()
+ String region = getS3Region()
+ String bucket = context.config.otherConfigs.get("s3BucketName");
+ String path = "regression/load_p0/tvf"
+
+ def tableName = "test_tvf_error_column"
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `k1` int NULL,
+ `k2` datetime(3) NOT NULL,
+ `v1` varchar(5) REPLACE NOT NULL,
+ `v2` varchar(5) REPLACE NULL,
+ `v3` varchar(5) REPLACE NULL,
+ `v4` varchar(5) REPLACE NULL
+ ) ENGINE=OLAP
+ aggregate KEY(`k1`,`k2`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 3
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1");
+ """
+
+ // Step1: date length overflow
+ test {
+ sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_data_with_length_limit.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "region" = "${region}"
+ );
+ """
+ exception "Insert has filtered data in strict mode"
+ }
+
+
+ // Step2: column with wrong type
+ test {
+ sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_column_with_wrong_type.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "region" = "${region}"
+ );
+ """
+ exception "Insert has filtered data in strict mode"
+ }
+
+ // Step3: NOT NULL column with NULL data
+ test {
+ sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_not_column_with_null_data.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "region" = "${region}"
+ );
+ """
+ exception "Insert has filtered data in strict mode"
+ }
+
+ // Step4: NOT NULL column with NULL data
+ test {
+ sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_not_null_column_with_empty_string_data.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "region" = "${region}"
+ );
+ """
+ exception "Insert has filtered data in strict mode"
+ }
+
+ // Step5: error compression type
+ test {
+ sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_data_with_varchar_quotatio.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "parquet",
+ "column_separator" = ",",
+ "region" = "${region}"
+ );
+ """
+ exception "Invalid magic number in parquet file"
+ }
+
+
+ // Step6: error compression type
+ test {
+ sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_data_with_varchar_quotatio.csv.zip";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "csv",
+ "compress" = "gz",
+ "column_separator" = ",",
+ "region" = "${region}"
+ );
+ """
+ exception "Only support csv data in utf8 codec"
+ }
+
+ // Step7: don't support lz4 at this moment
+ // test {
+ // sql """
+ // INSERT INTO ${tableName}
+ // SELECT * FROM S3 (
+ // "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_data_with_symbol.csv.gz";,
+ // "ACCESS_KEY"= "${ak}",
+ // "SECRET_KEY" = "${sk}",
+ // "format" = "csv",
+ // "compress" = "lz4".
+ // "column_separator" = ",",
+ // "region" = "${region}"
+ // );
+ // """
+ // exception "insert into cols should be corresponding to the query
output"
+ // }
+
+
+}
diff --git a/regression-test/suites/load_p0/tvf/test_tvf_error_data.groovy
b/regression-test/suites/load_p0/tvf/test_tvf_error_data.groovy
new file mode 100644
index 00000000000..b8fb7f99900
--- /dev/null
+++ b/regression-test/suites/load_p0/tvf/test_tvf_error_data.groovy
@@ -0,0 +1,202 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_tvf_error_data", "p0") {
+ String ak = getS3AK()
+ String sk = getS3SK()
+ String s3_endpoint = getS3Endpoint()
+ String region = getS3Region()
+ String bucket = context.config.otherConfigs.get("s3BucketName");
+ String path = "regression/load_p0/tvf"
+
+ def tableName = "test_tvf_error_data"
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ `k1` int NULL,
+ `k2` datetime(3) NULL,
+ `v1` varchar(50) NULL,
+ `v2` varchar(50) NULL,
+ `v3` varchar(50) NULL,
+ `v4` varchar(50) NULL
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k1`,`k2`)
+ COMMENT 'OLAP'
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 3
+ PROPERTIES ("replication_allocation" = "tag.location.default: 1");
+ """
+
+ // Step1: date with quotatio
+ sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_data_with_date_quotatio.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "region" = "${region}"
+ );
+ """
+ qt_select """
+ select * from ${tableName}
+ """
+ sql """ truncate table ${tableName} """
+
+ // Step2: use trim_double_quotes to ignore quotatio
+ sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_data_with_date_quotatio.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "trim_double_quotes" = "true",
+ "region" = "${region}"
+ );
+ """
+ qt_select """
+ select * from ${tableName}
+ """
+ sql """ truncate table ${tableName} """
+
+ // Step3: string with quotatio init
+ sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_data_with_varchar_quotatio.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "region" = "${region}"
+ );
+ """
+ qt_select """
+ select * from ${tableName}
+ """
+ sql """ truncate table ${tableName} """
+
+ // Step4: string with double quotatio init
+ qt_sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_data_with_varchar_double_quotatio.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "region" = "${region}"
+ );
+ """
+ qt_select """
+ select * from ${tableName}
+ """
+ sql """ truncate table ${tableName} """
+
+ // Step5: string with symbol init
+ qt_sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_data_with_symbol.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "region" = "${region}"
+ );
+ """
+ qt_select """
+ select * from ${tableName}
+ """
+ sql """ truncate table ${tableName} """
+
+ // Step6: string with enclose init
+ qt_sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_data_with_enclose.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "column_separator" = ",",
+ "format" = "csv",
+ "enclose" = "'",
+ "region" = "${region}"
+ );
+ """
+ qt_select """
+ select * from ${tableName}
+ """
+ sql """ truncate table ${tableName} """
+
+ // Step7: string with escape init
+ qt_sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_data_with_enclose.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "column_separator" = ",",
+ "format" = "csv",
+ "enclose" = "'",
+ "escape" = "\",
+ "region" = "${region}"
+ );
+ """
+ qt_select """
+ select * from ${tableName}
+ """
+ sql """ truncate table ${tableName} """
+
+
+ // Step8: string with separator init
+ qt_sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_data_with_separator.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "enclose" = "\\"",
+ "trim_double_quotes" = "true",
+ "region" = "${region}"
+ );
+ """
+ qt_select """
+ select * from ${tableName}
+ """
+ sql """ truncate table ${tableName} """
+
+ // Step9: string with separator init without enclose
+ test {
+ sql """
+ INSERT INTO ${tableName}
+ SELECT * FROM S3 (
+ "uri" =
"http://${bucket}.${s3_endpoint}/${path}/tvf_error_data_with_separator.csv";,
+ "ACCESS_KEY"= "${ak}",
+ "SECRET_KEY" = "${sk}",
+ "format" = "csv",
+ "column_separator" = ",",
+ "region" = "${region}"
+ );
+ """
+ exception "insert into cols should be corresponding to the query
output"
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
