This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3498453963767fe8443a194c295be1399907ca54 Author: amory <[email protected]> AuthorDate: Wed Apr 17 11:17:17 2024 +0800 [FIX](load)fix load with split-by-string (#33713) --- .../src/main/java/org/apache/doris/load/Load.java | 6 +-- .../stream_load/test_array_split_string.csv | 1 + .../stream_load/test_array_with_func_load.out | 4 ++ .../stream_load/test_array_with_func_load.groovy | 56 ++++++++++++++++++++++ 4 files changed, 64 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java index ed86310b435..d5c316aaa87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java @@ -550,11 +550,11 @@ public class Load { } } - // Array type do not support cast now + // Array/Map/Struct type do not support cast now Type exprReturnType = expr.getType(); - if (exprReturnType.isArrayType()) { + if (exprReturnType.isComplexType()) { Type schemaType = tbl.getColumn(entry.getKey()).getType(); - if (exprReturnType != schemaType) { + if (!exprReturnType.matchesType(schemaType)) { throw new AnalysisException("Don't support load from type:" + exprReturnType + " to type:" + schemaType + " for column:" + entry.getKey()); } diff --git a/regression-test/data/load_p0/stream_load/test_array_split_string.csv b/regression-test/data/load_p0/stream_load/test_array_split_string.csv new file mode 100644 index 00000000000..15101d6d45e --- /dev/null +++ b/regression-test/data/load_p0/stream_load/test_array_split_string.csv @@ -0,0 +1 @@ +1,A|B|C diff --git a/regression-test/data/load_p0/stream_load/test_array_with_func_load.out b/regression-test/data/load_p0/stream_load/test_array_with_func_load.out new file mode 100644 index 00000000000..a5968eefe64 --- /dev/null +++ b/regression-test/data/load_p0/stream_load/test_array_with_func_load.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !sql -- +1 ["A", "B", "C"] + diff --git a/regression-test/suites/load_p0/stream_load/test_array_with_func_load.groovy b/regression-test/suites/load_p0/stream_load/test_array_with_func_load.groovy new file mode 100644 index 00000000000..44e12a14fb2 --- /dev/null +++ b/regression-test/suites/load_p0/stream_load/test_array_with_func_load.groovy @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_array_with_func_load", "p0") { + def tableName = "test_array_split_string" + sql """ set enable_fallback_to_original_planner=false;""" + sql """ DROP TABLE IF EXISTS ${tableName} """ + sql """ CREATE TABLE ${tableName} ( + id int, + v1 array<VARCHAR(65533)> + ) ENGINE=OLAP + duplicate key (`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 1 + PROPERTIES ( + "replication_allocation" = "tag.location.default: 1", + "in_memory" = "false", + "storage_format" = "V2" + ); + """ + + streamLoad { + table "${tableName}" + + set 'column_separator', ',' + set 'columns', """ id,tmp,v1=split_by_string(tmp,'|') """ + file 'test_array_split_string.csv' + check { result, exception, startTime, endTime -> + if (exception != null) { + throw exception + } + log.info("Stream load result: ${result}".toString()) + def json = parseJson(result) + assertEquals("success", json.Status.toLowerCase()) + assertEquals(1, json.NumberLoadedRows) + } + } + + sql """sync""" + + qt_sql """select * from ${tableName} order by id;""" + +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
