This is an automated email from the ASF dual-hosted git repository.
mrhhsg pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c1b2bdb34d0 [fix](jsonb) Stop truncating JSON text at a U+0000 inside
a string (#68244)
c1b2bdb34d0 is described below
commit c1b2bdb34d06dd61f722cdc6ccc64fab6394b8a4
Author: Jerry Hu <[email protected]>
AuthorDate: Wed Sep 23 15:30:44 2026 +0800
[fix](jsonb) Stop truncating JSON text at a U+0000 inside a string (#68244)
### What problem does this PR solve?
Issue Number: None
Problem Summary: `JsonbToJson::string_to_json()` walks the string with
`ptr != str + len && *ptr`, so it stops at the first NUL byte even
though the length says the string is longer. U+0000 is a legal character
inside a JSON string and the parser stores it by length, so every path
that renders a JSONB value back to text loses everything after it:
```
SELECT k, v FROM (SELECT 1) d
LATERAL VIEW json_each_text(concat('{"x":["a', unhex('5C'), 'u0000b"]}')) t
AS k, v;
-- before: x ["a"]
-- after: x ["a\u0000b"]
SELECT c FROM (SELECT 1) d
LATERAL VIEW explode_json_array_string(concat('[["a', unhex('5C'),
'u0000b"]]')) t AS c;
-- before: ["a"]
-- after: ["a\u0000b"]
```
The same string as a direct array element is copied by length and
already keeps the NUL, which is why only nested complex values are
truncated.
Bound the loop by the length only. The NUL then reaches the escaping
branch and is written as `\u0000`, like any other control character.
### Release note
None
### Check List (For Author)
- Test:
- Unit Test: Yes, `JsonbParserTest.ParseJsonWithEscapedNulInString`,
`ParseJsonWithEscapedNulInNestedArray` and
`ParseJsonWithEscapedNulInKey` round trip a U+0000 through the JSON text
output.
- Regression test: Yes, `test_json_text_embedded_nul` covers
`json_each_text` and `explode_json_array_string` over nested values that
contain U+0000.
- Behavior changed: Yes, textualized JSON values keep everything after a
U+0000 and escape it as `\u0000` instead of being silently truncated.
- Does this need documentation: No
https://claude.ai/code/session_01RZ36Pij3o8fGnYYg33PKnq
---
be/src/util/jsonb_utils.h | 9 +++-
be/test/util/jsonb_parser_simd_test.cpp | 37 +++++++++++++
.../table_function/test_json_text_embedded_nul.out | 19 +++++++
.../test_json_text_embedded_nul.groovy | 62 ++++++++++++++++++++++
4 files changed, 125 insertions(+), 2 deletions(-)
diff --git a/be/src/util/jsonb_utils.h b/be/src/util/jsonb_utils.h
index abf05b4d311..05b2ab5ad08 100644
--- a/be/src/util/jsonb_utils.h
+++ b/be/src/util/jsonb_utils.h
@@ -107,8 +107,11 @@ private:
break;
}
case JsonbType::T_String: {
+ // getBlobLen() is the stored payload length. length() would drop a
+ // trailing NUL that belongs to the value, so it cannot be used
here;
+ // the writer stores strings by their exact length and never pads.
string_to_json(val->unpack<JsonbStringVal>()->getBlob(),
- val->unpack<JsonbStringVal>()->length());
+ val->unpack<JsonbStringVal>()->getBlobLen());
break;
}
case JsonbType::T_Binary: {
@@ -162,7 +165,9 @@ private:
return;
}
char char_buffer[16];
- for (const char* ptr = str; ptr != str + len && *ptr; ++ptr) {
+ // A JSON string may legally contain U+0000, so the loop must be
bounded by
+ // the length only; the NUL itself is escaped as \u0000 by the default
branch.
+ for (const char* ptr = str; ptr != str + len; ++ptr) {
if ((unsigned char)*ptr > 31 && *ptr != '\"' && *ptr != '\\') {
os_.put(*ptr);
} else {
diff --git a/be/test/util/jsonb_parser_simd_test.cpp
b/be/test/util/jsonb_parser_simd_test.cpp
index 53a666955ba..8f5b418a193 100644
--- a/be/test/util/jsonb_parser_simd_test.cpp
+++ b/be/test/util/jsonb_parser_simd_test.cpp
@@ -400,4 +400,41 @@ TEST_F(JsonbParserTest, ParseJsonWithToLongKey) {
EXPECT_FALSE(st.ok());
std::cout << st.msg() << std::endl;
}
+
+TEST_F(JsonbParserTest, ParseJsonWithEscapedNulInString) {
+ std::string_view json_with_nul = R"({"key":"a\u0000b"})";
+ std::string_view expected_json_with_nul = R"({"key":"a\u0000b"})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul),
Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithEscapedNulInNestedArray) {
+ std::string_view json_with_nul = R"({"key":["a\u0000b","c"]})";
+ std::string_view expected_json_with_nul = R"({"key":["a\u0000b","c"]})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul),
Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithTrailingNulInString) {
+ std::string_view json_with_nul = R"({"key":"a\u0000"})";
+ std::string_view expected_json_with_nul = R"({"key":"a\u0000"})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul),
Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithOnlyNulInString) {
+ std::string_view json_with_nul = R"({"key":"\u0000"})";
+ std::string_view expected_json_with_nul = R"({"key":"\u0000"})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul),
Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithTrailingNulInNestedArray) {
+ std::string_view json_with_nul = R"({"key":["a\u0000","\u0000"]})";
+ std::string_view expected_json_with_nul =
R"({"key":["a\u0000","\u0000"]})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul),
Status::OK());
+}
+
+TEST_F(JsonbParserTest, ParseJsonWithEscapedNulInKey) {
+ std::string_view json_with_nul = R"({"a\u0000b":1})";
+ std::string_view expected_json_with_nul = R"({"a\u0000b":1})";
+ EXPECT_EQ(parse_json_and_check(json_with_nul, expected_json_with_nul),
Status::OK());
+}
+
} // namespace doris
\ No newline at end of file
diff --git
a/regression-test/data/query_p0/sql_functions/table_function/test_json_text_embedded_nul.out
b/regression-test/data/query_p0/sql_functions/table_function/test_json_text_embedded_nul.out
new file mode 100644
index 00000000000..a3c8fb2de52
--- /dev/null
+++
b/regression-test/data/query_p0/sql_functions/table_function/test_json_text_embedded_nul.out
@@ -0,0 +1,19 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !json_each_text_nested_array --
+x 12 ["a\\u0000b"]
+
+-- !json_each_text_nested_object --
+x 16 {"y":"a\\u0000b"}
+
+-- !json_each_text_trailing_nul --
+x 20 ["a\\u0000","\\u0000"]
+
+-- !explode_json_array_string_direct --
+3 610062
+
+-- !explode_json_array_string_nested --
+12 ["a\\u0000b"]
+
+-- !explode_json_array_string_trailing_nul --
+11 ["a\\u0000"]
+
diff --git
a/regression-test/suites/query_p0/sql_functions/table_function/test_json_text_embedded_nul.groovy
b/regression-test/suites/query_p0/sql_functions/table_function/test_json_text_embedded_nul.groovy
new file mode 100644
index 00000000000..7e1e3beeea7
--- /dev/null
+++
b/regression-test/suites/query_p0/sql_functions/table_function/test_json_text_embedded_nul.groovy
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_json_text_embedded_nul") {
+ // unhex('5C') is a backslash, so every JSON literal below really contains
+ // \u0000, the legal escape for U+0000 inside a JSON string. Rendering such
+ // a nested value back to text must not stop at the NUL.
+
+ qt_json_each_text_nested_array '''
+ SELECT k, length(v), v
+ FROM (SELECT 1) d
+ LATERAL VIEW json_each_text(concat('{"x":["a', unhex('5C'),
'u0000b"]}')) t AS k, v
+ '''
+
+ qt_json_each_text_nested_object '''
+ SELECT k, length(v), v
+ FROM (SELECT 1) d
+ LATERAL VIEW json_each_text(concat('{"x":{"y":"a', unhex('5C'),
'u0000b"}}')) t AS k, v
+ '''
+
+ // A NUL at the very end of a string used to be trimmed by the
stored-length
+ // helper, so keep a dedicated case for it.
+ qt_json_each_text_trailing_nul '''
+ SELECT k, length(v), v
+ FROM (SELECT 1) d
+ LATERAL VIEW json_each_text(concat('{"x":["a', unhex('5C'),
'u0000","', unhex('5C'), 'u0000"]}')) t AS k, v
+ '''
+
+ // A direct string element is copied by length, so it keeps the raw NUL
byte;
+ // compare it through hex() to keep the expected output printable.
+ qt_explode_json_array_string_direct '''
+ SELECT length(c), hex(c)
+ FROM (SELECT 1) d
+ LATERAL VIEW explode_json_array_string(concat('["a', unhex('5C'),
'u0000b"]')) t AS c
+ '''
+
+ qt_explode_json_array_string_nested '''
+ SELECT length(c), c
+ FROM (SELECT 1) d
+ LATERAL VIEW explode_json_array_string(concat('[["a', unhex('5C'),
'u0000b"]]')) t AS c
+ '''
+
+ qt_explode_json_array_string_trailing_nul '''
+ SELECT length(c), c
+ FROM (SELECT 1) d
+ LATERAL VIEW explode_json_array_string(concat('[["a', unhex('5C'),
'u0000"]]')) t AS c
+ '''
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
