This is an automated email from the ASF dual-hosted git repository.
yangzhg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2a063355ad [fix](vstream load) Fix the default value insertion problem
when importing json (#12601)
2a063355ad is described below
commit 2a063355ad2afa91dd416722749adbefb98cd2bd
Author: yinzhijian <[email protected]>
AuthorDate: Fri Sep 16 09:54:45 2022 +0800
[fix](vstream load) Fix the default value insertion problem when importing
json (#12601)
* [fix](vstream load) Fix the default value insertion problem when
importing json
* update
---
be/src/vec/exec/vjson_scanner.cpp | 43 ++++++--
.../load_p0/stream_load/json_column_match.json | 62 ++++++++++++
...n_column_exclude_schema_without_jsonpath.groovy | 112 +++++++++++++++++++++
3 files changed, 207 insertions(+), 10 deletions(-)
diff --git a/be/src/vec/exec/vjson_scanner.cpp
b/be/src/vec/exec/vjson_scanner.cpp
index dbe6d986e4..b0f8ed5ba8 100644
--- a/be/src/vec/exec/vjson_scanner.cpp
+++ b/be/src/vec/exec/vjson_scanner.cpp
@@ -232,8 +232,9 @@ Status VJsonReader::_set_column_value(rapidjson::Value&
objectValue,
return Status::OK();
}
- int nullcount = 0;
int ctx_idx = 0;
+ bool has_valid_value = false;
+ size_t cur_row_count = columns[0]->size();
for (auto slot_desc : slot_descs) {
if (!slot_desc->is_materialized()) {
continue;
@@ -259,12 +260,11 @@ Status VJsonReader::_set_column_value(rapidjson::Value&
objectValue,
if (!(*valid)) {
return Status::OK();
}
+ has_valid_value = true;
} else { // not found
- if (slot_desc->is_nullable()) {
- auto* nullable_column =
reinterpret_cast<vectorized::ColumnNullable*>(column_ptr);
- nullable_column->insert_default();
- nullcount++;
- } else {
+ // When the entire row has no valid value, this row should be
filtered,
+ // so the default value cannot be directly inserted here
+ if (!slot_desc->is_nullable()) {
RETURN_IF_ERROR(_append_error_msg(
objectValue,
"The column `{}` is not nullable, but it's not found
in jsondata.",
@@ -273,12 +273,29 @@ Status VJsonReader::_set_column_value(rapidjson::Value&
objectValue,
}
}
}
-
- if (nullcount == slot_descs.size()) {
+ if (!has_valid_value) {
RETURN_IF_ERROR(_append_error_msg(objectValue, "All fields is null,
this is a invalid row.",
"", valid));
return Status::OK();
}
+ ctx_idx = 0;
+ int nullcount = 0;
+ // fill missing slot
+ for (auto slot_desc : slot_descs) {
+ if (!slot_desc->is_materialized()) {
+ continue;
+ }
+ int dest_index = ctx_idx++;
+ auto* column_ptr = columns[dest_index].get();
+ if (column_ptr->size() < cur_row_count + 1) {
+ DCHECK(column_ptr->size() == cur_row_count);
+ column_ptr->assume_mutable()->insert_default();
+ ++nullcount;
+ }
+ DCHECK(column_ptr->size() == cur_row_count + 1);
+ }
+ // There is at least one valid value here
+ DCHECK(nullcount < columns.size());
*valid = true;
return Status::OK();
}
@@ -583,6 +600,7 @@ Status
VSIMDJsonReader::_set_column_value(simdjson::ondemand::value objectValue,
auto object_val = objectValue.get_object();
size_t cur_row_count = block.rows();
+ bool has_valid_value = false;
// iterate through object, simdjson::ondemond will parsing on the fly
for (auto field : object_val) {
std::string_view key;
@@ -596,6 +614,11 @@ Status
VSIMDJsonReader::_set_column_value(simdjson::ondemand::value objectValue,
if (!(*valid)) {
return Status::OK();
}
+ has_valid_value = true;
+ }
+ if (!has_valid_value) {
+ RETURN_IF_ERROR(_append_error_msg("All fields is null, this is a
invalid row.", "", valid));
+ return Status::OK();
}
int nullcount = 0;
@@ -603,11 +626,11 @@ Status
VSIMDJsonReader::_set_column_value(simdjson::ondemand::value objectValue,
for (const auto& column_type_name : block) {
auto column = column_type_name.column;
if (column->size() < cur_row_count + 1) {
- assert(column->size() == cur_row_count);
+ DCHECK(column->size() == cur_row_count);
column->assume_mutable()->insert_default();
++nullcount;
}
- assert(column->size() == cur_row_count + 1);
+ DCHECK(column->size() == cur_row_count + 1);
}
if (nullcount == block.columns()) {
RETURN_IF_ERROR(_append_error_msg("All fields is null, this is a
invalid row.", "", valid));
diff --git a/regression-test/data/load_p0/stream_load/json_column_match.json
b/regression-test/data/load_p0/stream_load/json_column_match.json
new file mode 100644
index 0000000000..50a8aa0c6b
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/json_column_match.json
@@ -0,0 +1,62 @@
+ [ {"hh10": "7395.231067", "hh3": "1000", "hh2": "100", "hh1": "10", "hh0":
"1", "hh7": "ynqnzeowymt", "hh6": "t", "hh5": "2010-01-01", "hh4": "2011-01-01
00:00:00", "hh9": "180.998031", "hh8": "38.638843"},
+ {"hh10": "2080.504502", "hh3": "2000", "hh2": "200", "hh1": "20", "hh0":
"2", "hh7": "hfkfwlr", "hh6": "f", "hh5": "2010-01-02", "hh4": "2012-01-01
00:00:00", "hh9": "539.922834", "hh8": "506.044046"},
+ {"hh10": "4605.253205", "hh3": "3000", "hh2": "300", "hh1": "30", "hh0":
"3", "hh7": "uoclasp", "hh6": "t", "hh5": "2010-01-03", "hh4": "2013-01-01
00:00:00", "hh9": "577.044148", "hh8": "377.793209"},
+ {"hh10": "7291.703724", "hh3": "4000", "hh2": "400", "hh1": "40", "hh0":
"4", "hh7": "iswngzeodfhptjzgswsddt", "hh6": "n", "hh5": "2010-01-04", "hh4":
"2014-01-01 00:00:00", "hh9": "919.067864", "hh8": "871.354536"},
+ {"hh10": "3903.906901", "hh3": "5000", "hh2": "500", "hh1": "50", "hh0":
"5", "hh7": "sqodagzlyrmcelyxgcgcsfuxadcdt", "hh6": "a", "hh5": "2010-01-05",
"hh4": "2015-01-01 00:00:00", "hh9": "929.660783", "hh8": "462.067902"},
+ {"hh10": "6514.405051", "hh3": "6000", "hh2": "600", "hh1": "60", "hh0":
"6", "hh7": "obdrei", "hh6": "m", "hh5": "2010-01-06", "hh4": "2016-01-01
00:00:00", "hh9": "882.708491", "hh8": "921.867848"},
+ {"hh10": "8604.198677", "hh3": "7000", "hh2": "700", "hh1": "70", "hh0":
"7", "hh7": "cuobdhvrgkugknj", "hh6": "a", "hh5": "2010-01-07", "hh4":
"2017-01-01 00:00:00", "hh9": "209.420112", "hh8": "141.656421"},
+ {"hh10": "7784.859446", "hh3": "8000", "hh2": "800", "hh1": "80", "hh0":
"8", "hh7": "phcxztwgjllhmj", "hh6": "z", "hh5": "2010-01-08", "hh4":
"2018-01-01 00:00:00", "hh9": "285.664871", "hh8": "762.813376"},
+ {"hh10": "4846.735593", "hh3": "9000", "hh2": "900", "hh1": "90", "hh0":
"9", "hh7": "nbarqjwilbkelk", "hh6": "b", "hh5": "2010-01-09", "hh4":
"2019-01-01 00:00:00", "hh9": "535.285510", "hh8": "92.702403"},
+ {"hh10": "7996.434686", "hh3": "10000", "hh2": "1000", "hh1": "100",
"hh0": "10", "hh7": "zucprgdnlgzzfl", "hh6": "s", "hh5": "2010-01-10", "hh4":
"2020-01-01 00:00:00", "hh9": "155.861217", "hh8": "26.874738"},
+ {"hh10": "3821.278705", "hh3": "11000", "hh2": "1100", "hh1": "110",
"hh0": "11", "hh7": "zabbwiimqemk", "hh6": "j", "hh5": "2010-01-11", "hh4":
"2021-01-01 00:00:00", "hh9": "558.999245", "hh8": "369.819134"},
+ {"hh10": "9334.300973", "hh3": "12000", "hh2": "1200", "hh1": "120",
"hh0": "12", "hh7": "elvkhacywhdzrazcvyunkjajnx", "hh6": "d", "hh5":
"2010-01-12", "hh4": "2022-01-01 00:00:00", "hh9": "181.877653", "hh8":
"536.045037"},
+ {"hh10": "9103.804406", "hh3": "13000", "hh2": "1300", "hh1": "130",
"hh0": "13", "hh7": "cylgmxlmkkrkk", "hh6": "x", "hh5": "2010-01-13", "hh4":
"2023-01-01 00:00:00", "hh9": "909.667873", "hh8": "990.836863"},
+ {"hh10": "9823.062496", "hh3": "14000", "hh2": "1400", "hh1": "140",
"hh0": "14", "hh7": "vpzwml", "hh6": "s", "hh5": "2010-01-14", "hh4":
"2024-01-01 00:00:00", "hh9": "295.541079", "hh8": "763.615167"},
+ {"hh10": "5859.241974", "hh3": "15000", "hh2": "1500", "hh1": "150",
"hh0": "15", "hh7": "yvvstxajxtgrimmrveljjbwo", "hh6": "f", "hh5":
"2010-01-15", "hh4": "2025-01-01 00:00:00", "hh9": "391.329557", "hh8":
"380.419929"},
+ {"hh10": "9430.138013", "hh3": "16000", "hh2": "1600", "hh1": "160",
"hh0": "16", "hh7": "yxtowesbeqyejvpfhkixpdw", "hh6": "b", "hh5": "2010-01-16",
"hh4": "2026-01-01 00:00:00", "hh9": "328.666079", "hh8": "312.933953"},
+ {"hh10": "3135.292842", "hh3": "17000", "hh2": "1700", "hh1": "170",
"hh0": "17", "hh7": "nrrzcgygjplgttf", "hh6": "d", "hh5": "2010-01-17", "hh4":
"2027-01-01 00:00:00", "hh9": "799.528036", "hh8": "472.844369"},
+ {"hh10": "9855.813711", "hh3": "18000", "hh2": "1800", "hh1": "180",
"hh0": "18", "hh7": "wfknyaxplas", "hh6": "e", "hh5": "2010-01-18", "hh4":
"2028-01-01 00:00:00", "hh9": "844.025534", "hh8": "971.549437"},
+ {"hh10": "3086.573874", "hh3": "19000", "hh2": "1900", "hh1": "190",
"hh0": "19", "hh7": "qtznw", "hh6": "f", "hh5": "2010-01-19", "hh4":
"2029-01-01 00:00:00", "hh9": "502.584396", "hh8": "37.568959"},
+ {"hh10": "2698.863638", "hh3": "20000", "hh2": "2000", "hh1": "200",
"hh0": "20", "hh7": "nwznneizhtmzemy", "hh6": "i", "hh5": "2010-01-20", "hh4":
"2030-01-01 00:00:00", "hh9": "972.561324", "hh8": "632.512198"},
+ {"hh10": "5185.373039", "hh3": "21000", "hh2": "2100", "hh1": "210",
"hh0": "21", "hh7": "sqqeanrmafdoheeizljifwsj", "hh6": "q", "hh5":
"2010-01-21", "hh4": "2031-01-01 00:00:00", "hh9": "933.754744", "hh8":
"942.888716"},
+ {"hh10": "6221.348291", "hh3": "22000", "hh2": "2200", "hh1": "220",
"hh0": "22", "hh7": "tirlgqmwrchusfyxitxttvxlioknz", "hh6": "h", "hh5":
"2010-01-22", "hh4": "2032-01-01 00:00:00", "hh9": "69.032071", "hh8":
"654.955955"},
+ {"hh10": "7481.574657", "hh3": "23000", "hh2": "2300", "hh1": "230",
"hh0": "23", "hh7": "imnrusvqy", "hh6": "s", "hh5": "2010-01-23", "hh4":
"2033-01-01 00:00:00", "hh9": "24.626005", "hh8": "468.780338"},
+ {"hh10": "3585.208809", "hh3": "24000", "hh2": "2400", "hh1": "240",
"hh0": "24", "hh7": "irpbe", "hh6": "s", "hh5": "2010-01-24", "hh4":
"2034-01-01 00:00:00", "hh9": "396.750845", "hh8": "537.906124"},
+ {"hh10": "6982.796060", "hh3": "25000", "hh2": "2500", "hh1": "250",
"hh0": "25", "hh7": "drpjerrdlel", "hh6": "w", "hh5": "2010-01-25", "hh4":
"2035-01-01 00:00:00", "hh9": "202.007097", "hh8": "662.123247"},
+ {"hh10": "3046.839203", "hh3": "26000", "hh2": "2600", "hh1": "260",
"hh0": "26", "hh7": "vmzutsaifmlimicshgjpsvhiowjnq", "hh6": "v", "hh5":
"2010-01-26", "hh4": "2036-01-01 00:00:00", "hh9": "64.115478", "hh8":
"115.145598"},
+ {"hh10": "9921.135045", "hh3": "27000", "hh2": "2700", "hh1": "270",
"hh0": "27", "hh7": "yrxspxgcwgbnjnmqkcido", "hh6": "d", "hh5": "2010-01-27",
"hh4": "2037-01-01 00:00:00", "hh9": "519.383701", "hh8": "362.227207"},
+ {"hh10": "4633.258698", "hh3": "28000", "hh2": "2800", "hh1": "280",
"hh0": "28", "hh7": "udfzebgnnxfjnoujtvlib", "hh6": "w", "hh5": "2010-01-28",
"hh4": "2038-01-01 00:00:00", "hh9": "998.471600", "hh8": "443.030856"},
+ {"hh10": "5465.010345", "hh3": "29000", "hh2": "2900", "hh1": "290",
"hh0": "29", "hh7": "jfgvke", "hh6": "p", "hh5": "2010-01-29", "hh4":
"2039-01-01 00:00:00", "hh9": "248.265099", "hh8": "8.602754"},
+ {"hh10": "2579.668636", "hh3": "30000", "hh2": "3000", "hh1": "300",
"hh0": "30", "hh7": "tbuvpobzluhbwkljlhwnrkrhowybk", "hh6": "d", "hh5":
"2010-01-30", "hh4": "2040-01-01 00:00:00", "hh9": "976.266490", "hh8":
"949.575593"},
+ {"v6": "7395.231067", "k4": "1000", "k3": "100", "k2": "10", "k1": "1",
"v3": "ynqnzeowymt", "v2": "t", "v1": "2010-01-01", "k5": "2011-01-01
00:00:00", "v5": "180.998031", "v4": "38.638843"},
+ {"v6": "2080.504502", "k4": "2000", "k3": "200", "k2": "20", "k1": "2",
"v3": "hfkfwlr", "v2": "f", "v1": "2010-01-02", "k5": "2012-01-01 00:00:00",
"v5": "539.922834", "v4": "506.044046"},
+ {"v6": "4605.253205", "k4": "3000", "k3": "300", "k2": "30", "k1": "3",
"v3": "uoclasp", "v2": "t", "v1": "2010-01-03", "k5": "2013-01-01 00:00:00",
"v5": "577.044148", "v4": "377.793209"},
+ {"v6": "7291.703724", "k4": "4000", "k3": "400", "k2": "40", "k1": "4",
"v3": "iswngzeodfhptjzgswsddt", "v2": "n", "v1": "2010-01-04", "k5":
"2014-01-01 00:00:00", "v5": "919.067864", "v4": "871.354536"},
+ {"v6": "3903.906901", "k4": "5000", "k3": "500", "k2": "50", "k1": "5",
"v3": "sqodagzlyrmcelyxgcgcsfuxadcdt", "v2": "a", "v1": "2010-01-05", "k5":
"2015-01-01 00:00:00", "v5": "929.660783", "v4": "462.067902"},
+ {"v6": "6514.405051", "k4": "6000", "k3": "600", "k2": "60", "k1": "6",
"v3": "obdrei", "v2": "m", "v1": "2010-01-06", "k5": "2016-01-01 00:00:00",
"v5": "882.708491", "v4": "921.867848"},
+ {"v6": "8604.198677", "k4": "7000", "k3": "700", "k2": "70", "k1": "7",
"v3": "cuobdhvrgkugknj", "v2": "a", "v1": "2010-01-07", "k5": "2017-01-01
00:00:00", "v5": "209.420112", "v4": "141.656421"},
+ {"v6": "7784.859446", "k4": "8000", "k3": "800", "k2": "80", "k1": "8",
"v3": "phcxztwgjllhmj", "v2": "z", "v1": "2010-01-08", "k5": "2018-01-01
00:00:00", "v5": "285.664871", "v4": "762.813376"},
+ {"v6": "4846.735593", "k4": "9000", "k3": "900", "k2": "90", "k1": "9",
"v3": "nbarqjwilbkelk", "v2": "b", "v1": "2010-01-09", "k5": "2019-01-01
00:00:00", "v5": "535.285510", "v4": "92.702403"},
+ {"v6": "7996.434686", "k4": "10000", "k3": "1000", "k2": "100", "k1":
"10", "v3": "zucprgdnlgzzfl", "v2": "s", "v1": "2010-01-10", "k5": "2020-01-01
00:00:00", "v5": "155.861217", "v4": "26.874738"},
+ {"v6": "3821.278705", "k4": "11000", "k3": "1100", "k2": "110", "k1":
"11", "v3": "zabbwiimqemk", "v2": "j", "v1": "2010-01-11", "k5": "2021-01-01
00:00:00", "v5": "558.999245", "v4": "369.819134"},
+ {"v6": "9334.300973", "k4": "12000", "k3": "1200", "k2": "120", "k1":
"12", "v3": "elvkhacywhdzrazcvyunkjajnx", "v2": "d", "v1": "2010-01-12", "k5":
"2022-01-01 00:00:00", "v5": "181.877653", "v4": "536.045037"},
+ {"v6": "9103.804406", "k4": "13000", "k3": "1300", "k2": "130", "k1":
"13", "v3": "cylgmxlmkkrkk", "v2": "x", "v1": "2010-01-13", "k5": "2023-01-01
00:00:00", "v5": "909.667873", "v4": "990.836863"},
+ {"v6": "9823.062496", "k4": "14000", "k3": "1400", "k2": "140", "k1":
"14", "v3": "vpzwml", "v2": "s", "v1": "2010-01-14", "k5": "2024-01-01
00:00:00", "v5": "295.541079", "v4": "763.615167"},
+ {"v6": "5859.241974", "k4": "15000", "k3": "1500", "k2": "150", "k1":
"15", "v3": "yvvstxajxtgrimmrveljjbwo", "v2": "f", "v1": "2010-01-15", "k5":
"2025-01-01 00:00:00", "v5": "391.329557", "v4": "380.419929"},
+ {"v6": "9430.138013", "k4": "16000", "k3": "1600", "k2": "160", "k1":
"16", "v3": "yxtowesbeqyejvpfhkixpdw", "v2": "b", "v1": "2010-01-16", "k5":
"2026-01-01 00:00:00", "v5": "328.666079", "v4": "312.933953"},
+ {"v6": "3135.292842", "k4": "17000", "k3": "1700", "k2": "170", "k1":
"17", "v3": "nrrzcgygjplgttf", "v2": "d", "v1": "2010-01-17", "k5": "2027-01-01
00:00:00", "v5": "799.528036", "v4": "472.844369"},
+ {"v6": "9855.813711", "k4": "18000", "k3": "1800", "k2": "180", "k1":
"18", "v3": "wfknyaxplas", "v2": "e", "v1": "2010-01-18", "k5": "2028-01-01
00:00:00", "v5": "844.025534", "v4": "971.549437"},
+ {"v6": "3086.573874", "k4": "19000", "k3": "1900", "k2": "190", "k1":
"19", "v3": "qtznw", "v2": "f", "v1": "2010-01-19", "k5": "2029-01-01
00:00:00", "v5": "502.584396", "v4": "37.568959"},
+ {"v6": "2698.863638", "k4": "20000", "k3": "2000", "k2": "200", "k1":
"20", "v3": "nwznneizhtmzemy", "v2": "i", "v1": "2010-01-20", "k5": "2030-01-01
00:00:00", "v5": "972.561324", "v4": "632.512198"},
+ {"v6": "5185.373039", "k4": "21000", "k3": "2100", "k2": "210", "k1":
"21", "v3": "sqqeanrmafdoheeizljifwsj", "v2": "q", "v1": "2010-01-21", "k5":
"2031-01-01 00:00:00", "v5": "933.754744", "v4": "942.888716"},
+ {"v6": "6221.348291", "k4": "22000", "k3": "2200", "k2": "220", "k1":
"22", "v3": "tirlgqmwrchusfyxitxttvxlioknz", "v2": "h", "v1": "2010-01-22",
"k5": "2032-01-01 00:00:00", "v5": "69.032071", "v4": "654.955955"},
+ {"v6": "7481.574657", "k4": "23000", "k3": "2300", "k2": "230", "k1":
"23", "v3": "imnrusvqy", "v2": "s", "v1": "2010-01-23", "k5": "2033-01-01
00:00:00", "v5": "24.626005", "v4": "468.780338"},
+ {"v6": "3585.208809", "k4": "24000", "k3": "2400", "k2": "240", "k1":
"24", "v3": "irpbe", "v2": "s", "v1": "2010-01-24", "k5": "2034-01-01
00:00:00", "v5": "396.750845", "v4": "537.906124"},
+ {"v6": "6982.796060", "k4": "25000", "k3": "2500", "k2": "250", "k1":
"25", "v3": "drpjerrdlel", "v2": "w", "v1": "2010-01-25", "k5": "2035-01-01
00:00:00", "v5": "202.007097", "v4": "662.123247"},
+ {"v6": "3046.839203", "k4": "26000", "k3": "2600", "k2": "260", "k1":
"26", "v3": "vmzutsaifmlimicshgjpsvhiowjnq", "v2": "v", "v1": "2010-01-26",
"k5": "2036-01-01 00:00:00", "v5": "64.115478", "v4": "115.145598"},
+ {"v6": "9921.135045", "k4": "27000", "k3": "2700", "k2": "270", "k1":
"27", "v3": "yrxspxgcwgbnjnmqkcido", "v2": "d", "v1": "2010-01-27", "k5":
"2037-01-01 00:00:00", "v5": "519.383701", "v4": "362.227207"},
+ {"v6": "4633.258698", "k4": "28000", "k3": "2800", "k2": "280", "k1":
"28", "v3": "udfzebgnnxfjnoujtvlib", "v2": "w", "v1": "2010-01-28", "k5":
"2038-01-01 00:00:00", "v5": "998.471600", "v4": "443.030856"},
+ {"v6": "5465.010345", "k4": "29000", "k3": "2900", "k2": "290", "k1":
"29", "v3": "jfgvke", "v2": "p", "v1": "2010-01-29", "k5": "2039-01-01
00:00:00", "v5": "248.265099", "v4": "8.602754"},
+ {"v6": "2579.668636", "k4": "30000", "k3": "3000", "k2": "300", "k1":
"30", "v3": "tbuvpobzluhbwkljlhwnrkrhowybk", "v2": "d", "v1": "2010-01-30",
"k5": "2040-01-01 00:00:00", "v5": "976.266490", "v4": "949.575593"}
+]
+
diff --git
a/regression-test/suites/load_p0/stream_load/load_json_column_exclude_schema_without_jsonpath.groovy
b/regression-test/suites/load_p0/stream_load/load_json_column_exclude_schema_without_jsonpath.groovy
new file mode 100644
index 0000000000..fbb64e29d0
--- /dev/null
+++
b/regression-test/suites/load_p0/stream_load/load_json_column_exclude_schema_without_jsonpath.groovy
@@ -0,0 +1,112 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_load_json_column_exclude_schema_without_jsonpath", "p0") {
+ // define a sql table
+ def testTable = "tbl_test_json_load"
+ def dbName = "test_query_db"
+
+ def create_test_table = {enable_vectorized_flag ->
+ if (enable_vectorized_flag) {
+ sql """ set enable_vectorized_engine = true """
+ } else {
+ sql """ set enable_vectorized_engine = false """
+ }
+
+ def result1 = sql """
+ CREATE TABLE IF NOT EXISTS ${testTable} (
+ k1 TINYINT NULL,
+ k2 SMALLINT NULL,
+ k3 INT NULL,
+ k4 BIGINT NULL,
+ k5 DATETIME NULL,
+ v1 DATE REPLACE NULL,
+ v2 CHAR REPLACE NULL,
+ v3 VARCHAR(4096) REPLACE NULL,
+ v4 FLOAT SUM NULL,
+ v5 DOUBLE SUM NULL,
+ v6 DECIMAL(20,7) SUM NULL
+ ) ENGINE=OLAP
+ AGGREGATE KEY(k1,k2,k3,k4,k5)
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 5
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "storage_format" = "V2"
+ )
+ """
+ }
+
+ def load_array_data = {table_name, strip_flag, read_flag, format_flag,
exprs, json_paths,
+ json_root, where_expr, fuzzy_flag, column_sep,
file_name ->
+ // load the json data
+ streamLoad {
+ table table_name
+
+ // set http request header params
+ set 'strip_outer_array', strip_flag
+ set 'read_json_by_line', read_flag
+ set 'format', format_flag
+ set 'columns', exprs
+ set 'jsonpaths', json_paths
+ set 'json_root', json_root
+ set 'where', where_expr
+ set 'fuzzy_parse', fuzzy_flag
+ set 'column_separator', column_sep
+ set 'max_filter_ratio', '0.4'
+ file file_name // import json file
+ time 10000 // limit inflight 10s
+
+ // if declared a check callback, the default check condition will
ignore.
+ // So you must check all condition
+ check { result, exception, startTime, endTime ->
+ if (exception != null) {
+ throw exception
+ }
+ log.info("Stream load result: ${result}".toString())
+ def json = parseJson(result)
+ assertEquals("fail", json.Status.toLowerCase())
+ assertEquals(json.NumberTotalRows, json.NumberLoadedRows +
json.NumberFilteredRows)
+ assertTrue(json.NumberLoadedRows == 30);
+ assertTrue(json.NumberFilteredRows == 30);
+ }
+ }
+ }
+
+ // case1: import array data in json format and enable vectorized engine
+ try {
+ sql "DROP TABLE IF EXISTS ${testTable}"
+
+ create_test_table.call(true)
+
+ load_array_data.call(testTable, 'true', '', 'json', '', '', '', '',
'', '', 'json_column_match.json')
+
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${testTable}")
+ }
+
+ // case2: import array data in json format and disable vectorized engine
+ try {
+ sql "DROP TABLE IF EXISTS ${testTable}"
+
+ create_test_table.call(false)
+
+ load_array_data.call(testTable, 'true', '', 'json', '', '', '', '',
'', '', 'json_column_match.json')
+
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${testTable}")
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
