This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new d9fd419e47e [Fix](JsonReader) fix json with duplicate key entry may
result out of bound exception (#38147)
d9fd419e47e is described below
commit d9fd419e47eec8dca292a32642c42dee6f9e2585
Author: lihangyu <[email protected]>
AuthorDate: Fri Jul 19 22:53:02 2024 +0800
[Fix](JsonReader) fix json with duplicate key entry may result out of
bound exception (#38147)
#38146
---
be/src/vec/exec/format/json/new_json_reader.cpp | 3 +++
.../stream_load/test_duplicate_json_keys.json | 1 +
.../data/load_p0/stream_load/test_json_load.out | 2 ++
.../load_p0/stream_load/test_json_load.groovy | 25 ++++++++++++++++++++++
4 files changed, 31 insertions(+)
diff --git a/be/src/vec/exec/format/json/new_json_reader.cpp
b/be/src/vec/exec/format/json/new_json_reader.cpp
index 5636e242761..fabb699d6f3 100644
--- a/be/src/vec/exec/format/json/new_json_reader.cpp
+++ b/be/src/vec/exec/format/json/new_json_reader.cpp
@@ -1349,6 +1349,9 @@ Status
NewJsonReader::_simdjson_set_column_value(simdjson::ondemand::object* val
// This key is not exist in slot desc, just ignore
continue;
}
+ if (_seen_columns[column_index]) {
+ continue;
+ }
simdjson::ondemand::value val = field.value();
auto* column_ptr =
block.get_by_position(column_index).column->assume_mutable().get();
RETURN_IF_ERROR(
diff --git
a/regression-test/data/load_p0/stream_load/test_duplicate_json_keys.json
b/regression-test/data/load_p0/stream_load/test_duplicate_json_keys.json
new file mode 100644
index 00000000000..4bf1079cf22
--- /dev/null
+++ b/regression-test/data/load_p0/stream_load/test_duplicate_json_keys.json
@@ -0,0 +1 @@
+{"k1" : 10, "k1" : 100}
\ No newline at end of file
diff --git a/regression-test/data/load_p0/stream_load/test_json_load.out
b/regression-test/data/load_p0/stream_load/test_json_load.out
index 9f064572f13..588b6edb004 100644
--- a/regression-test/data/load_p0/stream_load/test_json_load.out
+++ b/regression-test/data/load_p0/stream_load/test_json_load.out
@@ -248,3 +248,5 @@ android \N \N \N \N \N
-- !select28 --
test k2_value
+-- !select29 --
+10 \N
diff --git a/regression-test/suites/load_p0/stream_load/test_json_load.groovy
b/regression-test/suites/load_p0/stream_load/test_json_load.groovy
index da6bea1afcd..7f182a44b74 100644
--- a/regression-test/suites/load_p0/stream_load/test_json_load.groovy
+++ b/regression-test/suites/load_p0/stream_load/test_json_load.groovy
@@ -853,4 +853,29 @@ suite("test_json_load", "p0") {
} finally {
try_sql("DROP TABLE IF EXISTS ${testTable}")
}
+
+ // add duplicate json entry case
+ try {
+ sql "DROP TABLE IF EXISTS ${testTable}"
+ sql """CREATE TABLE IF NOT EXISTS ${testTable}
+ (
+ `k1` varchar(1024) NULL,
+ `k2` varchar(1024) NULL
+ )
+ DUPLICATE KEY(`k1`)
+ COMMENT ''
+ DISTRIBUTED BY RANDOM BUCKETS 1
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );"""
+
+ load_json_data.call("${testTable}", "${testTable}_case29", 'false',
'true', 'json', '', '',
+ '', '', '', 'test_duplicate_json_keys.json',
false, 1)
+
+ sql "sync"
+ qt_select29 "select * from ${testTable}"
+
+ } finally {
+ try_sql("DROP TABLE IF EXISTS ${testTable}")
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
