This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ce79711b0da [FIX](serde) fix map/array deserialize string with quote
pair (#24808)
ce79711b0da is described below
commit ce79711b0dad3f5237a151d099257468628f52b7
Author: amory <[email protected]>
AuthorDate: Sat Sep 23 01:12:20 2023 +0800
[FIX](serde) fix map/array deserialize string with quote pair (#24808)
---
.../vec/data_types/serde/data_type_array_serde.cpp | 9 +-
.../vec/data_types/serde/data_type_map_serde.cpp | 9 +-
.../stream_load/test_map_load_and_function.out | 102 ++++++++++-----------
3 files changed, 67 insertions(+), 53 deletions(-)
diff --git a/be/src/vec/data_types/serde/data_type_array_serde.cpp
b/be/src/vec/data_types/serde/data_type_array_serde.cpp
index 15501aeaac2..b73c2ef9a5d 100644
--- a/be/src/vec/data_types/serde/data_type_array_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_array_serde.cpp
@@ -108,10 +108,17 @@ Status
DataTypeArraySerDe::deserialize_one_cell_from_json(IColumn& column, Slice
slices.emplace_back(slice);
size_t slice_size = slice.size;
// pre add total slice can reduce lasted element check.
+ char quote_char = 0;
for (int idx = 0; idx < slice_size; ++idx) {
char c = slice[idx];
if (c == '"' || c == '\'') {
- has_quote = !has_quote;
+ if (!has_quote) {
+ quote_char = c;
+ has_quote = !has_quote;
+ } else if (has_quote && quote_char == c) {
+ quote_char = 0;
+ has_quote = !has_quote;
+ }
} else if (!has_quote && (c == '[' || c == '{')) {
++nested_level;
} else if (!has_quote && (c == ']' || c == '}')) {
diff --git a/be/src/vec/data_types/serde/data_type_map_serde.cpp
b/be/src/vec/data_types/serde/data_type_map_serde.cpp
index 41db3236dc6..935c810dc86 100644
--- a/be/src/vec/data_types/serde/data_type_map_serde.cpp
+++ b/be/src/vec/data_types/serde/data_type_map_serde.cpp
@@ -216,10 +216,17 @@ Status
DataTypeMapSerDe::deserialize_one_cell_from_json(IColumn& column, Slice&
bool key_added = false;
int idx = 0;
int elem_deserialized = 0;
+ char quote_char = 0;
for (; idx < slice_size; ++idx) {
char c = slice[idx];
if (c == '"' || c == '\'') {
- has_quote = !has_quote;
+ if (!has_quote) {
+ quote_char = c;
+ has_quote = !has_quote;
+ } else if (has_quote && quote_char == c) {
+ quote_char = 0;
+ has_quote = !has_quote;
+ }
} else if (c == '\\' && idx + 1 < slice_size) { //escaped
++idx;
} else if (!has_quote && (c == '[' || c == '{')) {
diff --git
a/regression-test/data/load_p0/stream_load/test_map_load_and_function.out
b/regression-test/data/load_p0/stream_load/test_map_load_and_function.out
index 4b742ac8532..4b61206fae3 100644
--- a/regression-test/data/load_p0/stream_load/test_map_load_and_function.out
+++ b/regression-test/data/load_p0/stream_load/test_map_load_and_function.out
@@ -10,7 +10,7 @@
8 {}
9 {" 1,amy ":2, " k2 ":90, " k7 ":33}
10 {}
-11 {""k1'":4, "k2":400}
+11 {}
12 {"k3":23, NULL:20, "k4":NULL}
13 {"null":1}
15 \N
@@ -27,7 +27,7 @@
8 \N
9 \N
10 \N
-11 400
+11 \N
12 \N
13 \N
15 \N
@@ -46,7 +46,7 @@
8 {}
9 {" 1,amy ":2, " k2 ":90, " k7 ":33}
10 {}
-11 {""k1'":4, "k2":400}
+11 {}
12 {"k3":23, NULL:20, "k4":NULL}
13 {"null":1}
15 \N
@@ -125,7 +125,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} \N
10 {} \N
-11 {""k1'":4, "k2":400} \N
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} \N
13 {"null":1} \N
15 \N \N
@@ -144,7 +144,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} \N
10 {} \N
-11 {""k1'":4, "k2":400} 400
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} \N
13 {"null":1} \N
15 \N \N
@@ -163,7 +163,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} \N
10 {} \N
-11 {""k1'":4, "k2":400} \N
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} \N
13 {"null":1} \N
15 \N \N
@@ -182,7 +182,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} \N
10 {} \N
-11 {""k1'":4, "k2":400} \N
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} \N
13 {"null":1} \N
15 \N \N
@@ -201,7 +201,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} \N
10 {} \N
-11 {""k1'":4, "k2":400} \N
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} \N
13 {"null":1} \N
15 \N \N
@@ -220,7 +220,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} \N
10 {} \N
-11 {""k1'":4, "k2":400} \N
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} \N
13 {"null":1} \N
15 \N \N
@@ -239,7 +239,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} \N
10 {} \N
-11 {""k1'":4, "k2":400} \N
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} \N
13 {"null":1} \N
15 \N \N
@@ -258,7 +258,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} \N
10 {} \N
-11 {""k1'":4, "k2":400} \N
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} \N
13 {"null":1} \N
15 \N \N
@@ -277,7 +277,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} 2
10 {} \N
-11 {""k1'":4, "k2":400} \N
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} \N
13 {"null":1} \N
15 \N \N
@@ -296,7 +296,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} 90
10 {} \N
-11 {""k1'":4, "k2":400} \N
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} \N
13 {"null":1} \N
15 \N \N
@@ -315,7 +315,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} \N
10 {} \N
-11 {""k1'":4, "k2":400} \N
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} \N
13 {"null":1} 1
15 \N \N
@@ -334,7 +334,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} \N
10 {} \N
-11 {""k1'":4, "k2":400} \N
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} \N
13 {"null":1} \N
15 \N \N
@@ -353,7 +353,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} \N
10 {} \N
-11 {""k1'":4, "k2":400} \N
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} 20
13 {"null":1} \N
15 \N \N
@@ -372,7 +372,7 @@ k22
8 {} \N
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} \N
10 {} \N
-11 {""k1'":4, "k2":400} \N
+11 {} \N
12 {"k3":23, NULL:20, "k4":NULL} \N
13 {"null":1} \N
15 \N \N
@@ -394,7 +394,7 @@ k22
8 {} 0
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} 3
10 {} 0
-11 {""k1'":4, "k2":400} 2
+11 {} 0
12 {"k3":23, NULL:20, "k4":NULL} 3
13 {"null":1} 1
15 \N \N
@@ -428,7 +428,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -447,7 +447,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} true
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -466,7 +466,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -485,7 +485,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -504,7 +504,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -523,7 +523,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -542,7 +542,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -561,7 +561,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -580,7 +580,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} true
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -599,7 +599,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} true
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -618,7 +618,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} true
15 \N \N
@@ -637,7 +637,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -656,7 +656,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} true
13 {"null":1} false
15 \N \N
@@ -675,7 +675,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -706,7 +706,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} true
13 {"null":1} false
15 \N \N
@@ -725,7 +725,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} true
13 {"null":1} false
15 \N \N
@@ -744,7 +744,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -763,7 +763,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -782,7 +782,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -801,7 +801,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -820,7 +820,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} true
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -839,7 +839,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} true
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -858,7 +858,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} true
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -877,7 +877,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -896,7 +896,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} true
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -915,7 +915,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} true
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -934,7 +934,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} true
15 \N \N
@@ -953,7 +953,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} true
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -972,7 +972,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -991,7 +991,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} false
13 {"null":1} false
15 \N \N
@@ -1010,7 +1010,7 @@ false
8 {} false
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} false
10 {} false
-11 {""k1'":4, "k2":400} false
+11 {} false
12 {"k3":23, NULL:20, "k4":NULL} true
13 {"null":1} false
15 \N \N
@@ -1032,7 +1032,7 @@ false
8 {} []
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} [" 1,amy ", " k2 ", " k7 "]
10 {} []
-11 {""k1'":4, "k2":400} [""k1'", "k2"]
+11 {} []
12 {"k3":23, NULL:20, "k4":NULL} ["k3", NULL, "k4"]
13 {"null":1} ["null"]
15 \N \N
@@ -1054,7 +1054,7 @@ false
8 {} []
9 {" 1,amy ":2, " k2 ":90, " k7 ":33} [2, 90, 33]
10 {} []
-11 {""k1'":4, "k2":400} [4, 400]
+11 {} []
12 {"k3":23, NULL:20, "k4":NULL} [23, 20, NULL]
13 {"null":1} [1]
15 \N \N
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
