This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 353f9e3782 [regression](json) add a nullable case for stream load with
json format (#12505)
353f9e3782 is described below
commit 353f9e37825d135dee11b9993b8529fbda2c7ed7
Author: yinzhijian <[email protected]>
AuthorDate: Tue Sep 13 10:45:01 2022 +0800
[regression](json) add a nullable case for stream load with json format
(#12505)
---
.../stream_load/load_json_null_to_nullable.out | 13 +++
.../data/load_p0/stream_load/test_char.json | 9 ++
.../stream_load/load_json_null_to_nullable.groovy | 109 +++++++++++++++++++++
3 files changed, 131 insertions(+)
diff --git
a/regression-test/data/load_p0/stream_load/load_json_null_to_nullable.out
b/regression-test/data/load_p0/stream_load/load_json_null_to_nullable.out
new file mode 100644
index 0000000000..d564b5b99a
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/load_json_null_to_nullable.out
@@ -0,0 +1,13 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !select --
+\N \N
+
+H H
+h h
+
+-- !select --
+\N \N
+
+H H
+h h
+
diff --git a/regression-test/data/load_p0/stream_load/test_char.json
b/regression-test/data/load_p0/stream_load/test_char.json
new file mode 100644
index 0000000000..7cd7d69d9b
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/test_char.json
@@ -0,0 +1,9 @@
+[
+ {"k1": "h", "v1": "h"},
+ {"k1": "hello", "v1": "hello"},
+ {"k1": "hello,hello", "v1": "hello,hello"},
+ {"k1": "仓库", "v1": "安全"},
+ {"k1": "H", "v1": "H"},
+ {"k1": "", "v1": ""},
+ {"k1": null, "v1": null}
+]
diff --git
a/regression-test/suites/load_p0/stream_load/load_json_null_to_nullable.groovy
b/regression-test/suites/load_p0/stream_load/load_json_null_to_nullable.groovy
new file mode 100644
index 0000000000..e9a5665e03
--- /dev/null
+++
b/regression-test/suites/load_p0/stream_load/load_json_null_to_nullable.groovy
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_load_json_null_to_nullable", "p0") {
+ // define a sql table
+ def testTable = "tbl_test_json_load"
+ def dbName = "test_query_db"
+
+ def create_test_table = {enable_vectorized_flag ->
+ if (enable_vectorized_flag) {
+ sql """ set enable_vectorized_engine = true """
+ } else {
+ sql """ set enable_vectorized_engine = false """
+ }
+
+ def result1 = sql """
+ CREATE TABLE IF NOT EXISTS ${testTable} (
+ `k1` CHAR NULL COMMENT "",
+ `v1` CHAR NULL COMMENT ""
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`k1`)
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2"
+ )
+ """
+ }
+
+ def load_array_data = {table_name, strip_flag, read_flag, format_flag,
exprs, json_paths,
+ json_root, where_expr, fuzzy_flag, column_sep,
file_name ->
+ // load the json data
+ streamLoad {
+ table table_name
+
+ // set http request header params
+ set 'strip_outer_array', strip_flag
+ set 'read_json_by_line', read_flag
+ set 'format', format_flag
+ set 'columns', exprs
+ set 'jsonpaths', json_paths
+ set 'json_root', json_root
+ set 'where', where_expr
+ set 'fuzzy_parse', fuzzy_flag
+ set 'column_separator', column_sep
+ set 'max_filter_ratio', '1'
+ file file_name // import json file
+ time 10000 // limit inflight 10s
+
+ // if declared a check callback, the default check condition will
ignore.
+ // So you must check all condition
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("success", json.Status.toLowerCase())
+ assertEquals(json.NumberTotalRows, json.NumberLoadedRows +
json.NumberUnselectedRows
+ + json.NumberFilteredRows)
+ assertTrue(json.NumberLoadedRows > 0 && json.LoadBytes > 0)
+ }
+ }
+ }
+
+ // case1: import array data in json format and enable vectorized engine
+ try {
+ sql "DROP TABLE IF EXISTS ${testTable}"
+
+ create_test_table.call(true)
+
+ load_array_data.call(testTable, 'true', '', 'json', '', '', '', '',
'', '', 'test_char.json')
+
+ // select the table and check whether the data is correct
+ qt_select "select * from ${testTable} order by k1"
+
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${testTable}")
+ }
+
+ // case2: import array data in json format and disable vectorized engine
+ try {
+ sql "DROP TABLE IF EXISTS ${testTable}"
+
+ create_test_table.call(false)
+
+ load_array_data.call(testTable, 'true', '', 'json', '', '', '', '',
'', '', 'test_char.json')
+
+ // select the table and check whether the data is correct
+ qt_select "select * from ${testTable} order by k1"
+
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${testTable}")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
