This is an automated email from the ASF dual-hosted git repository.
snuyanzin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 2c768cde3ba [FLINK-40852][table] Treat `NULL` allowDuplicateKeys as
false in `PARSE_JSON` and `TRY_PARSE_JSON`
2c768cde3ba is described below
commit 2c768cde3bacf9e09b9791fdd91e4f3731614c36
Author: Moritz Manner <[email protected]>
AuthorDate: Wed Oct 7 13:25:24 2026 +0200
[FLINK-40852][table] Treat `NULL` allowDuplicateKeys as false in
`PARSE_JSON` and `TRY_PARSE_JSON`
---
docs/data/sql_functions.yml | 10 ++---
docs/data/sql_functions_zh.yml | 14 ++++---
.../functions/BuiltInFunctionDefinitions.java | 4 +-
.../planner/functions/JsonFunctionsITCase.java | 44 +++++++++++++++++++---
.../functions/scalar/ParseJsonFunction.java | 7 +++-
.../functions/scalar/TryParseJsonFunction.java | 7 +++-
6 files changed, 65 insertions(+), 21 deletions(-)
diff --git a/docs/data/sql_functions.yml b/docs/data/sql_functions.yml
index ee94bf1794f..c53b6767ade 100644
--- a/docs/data/sql_functions.yml
+++ b/docs/data/sql_functions.yml
@@ -1371,7 +1371,7 @@ variant:
If there are duplicate keys in the input JSON string, when
`allowDuplicateKeys` is true, the
parser will keep the last occurrence of all fields with the same key,
otherwise when
`allowDuplicateKeys` is false it will throw an error. The default value
of
- `allowDuplicateKeys` is false.
+ `allowDuplicateKeys` is false. A NULL `allowDuplicateKeys` is treated as
false.
See the `VARIANT` data type documentation for how JSON values are mapped
to variant kinds.
- sql: TRY_PARSE_JSON(json_string[, allow_duplicate_keys])
@@ -1380,10 +1380,10 @@ variant:
Try to parse a JSON string into a Variant if possible. If the JSON
string is invalid, return
NULL. To throw an error instead of returning NULL, use the `PARSE_JSON`
function.
- If there are duplicate keys in the input JSON string, when
`allowDuplicateKeys` is true, the
- parser will keep the last occurrence of all fields with the same key,
otherwise when
- `allowDuplicateKeys` is false it will throw an error. The default value
of
- `allowDuplicateKeys` is false.
+ If there are duplicate keys in the input JSON string, when
`allowDuplicateKeys` is true, the
+ parser will keep the last occurrence of all fields with the same key,
otherwise when
+ `allowDuplicateKeys` is false it will return NULL. The default value of
+ `allowDuplicateKeys` is false. A NULL `allowDuplicateKeys` is treated as
false.
See the `VARIANT` data type documentation for how JSON values are mapped
to variant kinds.
- sql: JSON_STRING(variant)
diff --git a/docs/data/sql_functions_zh.yml b/docs/data/sql_functions_zh.yml
index a6c6fa074b3..7c0a4e95d15 100644
--- a/docs/data/sql_functions_zh.yml
+++ b/docs/data/sql_functions_zh.yml
@@ -1461,9 +1461,10 @@ variant:
将 JSON 字符串解析为 Variant。如果 JSON 字符串无效，将抛出错误。如果希望返回 NULL 而不是抛出错误，
请使用 `TRY_PARSE_JSON` 函数。
- 如果输入的 JSON 字符串中存在重复的键，当 allowDuplicateKeys 为 true 时，解析器会保留最后一个出现的具有相
- 同键的字段，否则当 allowDuplicateKeys 为 false 时，它会抛出一个错误。默认情况下，
- allowDuplicateKeys 的值为 false。
+ If there are duplicate keys in the input JSON string, when
`allowDuplicateKeys` is true, the
+ parser will keep the last occurrence of all fields with the same key,
otherwise when
+ `allowDuplicateKeys` is false it will throw an error. The default value
of
+ `allowDuplicateKeys` is false. A NULL `allowDuplicateKeys` is treated as
false.
See the `VARIANT` data type documentation for how JSON values are mapped
to variant kinds.
@@ -1473,9 +1474,10 @@ variant:
尽可能将 JSON 字符串解析为 Variant。如果 JSON 字符串无效，则返回 NULL。如果希望抛出错误而不是返回 NULL，
请使用 `PARSE_JSON` 函数。
- 如果输入的 JSON 字符串中存在重复的键，当 allowDuplicateKeys 为 true 时，解析器会保留最后一个出现的具有相
- 同键的字段，否则当 allowDuplicateKeys 为 false 时，它会抛出一个错误。默认情况下，
- allowDuplicateKeys 的值为 false。
+ If there are duplicate keys in the input JSON string, when
`allowDuplicateKeys` is true, the
+ parser will keep the last occurrence of all fields with the same key,
otherwise when
+ `allowDuplicateKeys` is false it will return NULL. The default value of
+ `allowDuplicateKeys` is false. A NULL `allowDuplicateKeys` is treated as
false.
See the `VARIANT` data type documentation for how JSON values are mapped
to variant kinds.
diff --git
a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
index 08cc761375b..0bcb4c56649 100644
---
a/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
+++
b/flink-table/flink-table-common/src/main/java/org/apache/flink/table/functions/BuiltInFunctionDefinitions.java
@@ -3190,7 +3190,9 @@ public final class BuiltInFunctionDefinitions {
sequence(
logical(LogicalTypeFamily.CHARACTER_STRING),
logical(LogicalTypeRoot.BOOLEAN))))
-
.outputTypeStrategy(nullableIfArgs(explicit(DataTypes.VARIANT())))
+ .outputTypeStrategy(
+ nullableIfArgs(
+ ConstantArgumentCount.of(0),
explicit(DataTypes.VARIANT())))
.runtimeClass(
"org.apache.flink.table.runtime.functions.scalar.ParseJsonFunction")
.build();
diff --git
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/JsonFunctionsITCase.java
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/JsonFunctionsITCase.java
index 3d8d6eefa4e..2cd54053210 100644
---
a/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/JsonFunctionsITCase.java
+++
b/flink-table/flink-table-planner/src/test/java/org/apache/flink/table/planner/functions/JsonFunctionsITCase.java
@@ -1036,8 +1036,13 @@ class JsonFunctionsITCase extends
BuiltInFunctionTestBase {
// The bulk of parsing behavior is covered by
BinaryVariantInternalBuilderTest.
return List.of(
TestSetSpec.forFunction(BuiltInFunctionDefinitions.PARSE_JSON)
- .onFieldsWithData("{\"a\":1,\"b\":[2,3]}", "1e400",
"{\"a\":1,\"a\":2}")
- .andDataTypes(STRING().notNull(), STRING().notNull(),
STRING().notNull())
+ .onFieldsWithData(
+ "{\"a\":1,\"b\":[2,3]}", "1e400",
"{\"a\":1,\"a\":2}", null)
+ .andDataTypes(
+ STRING().notNull(),
+ STRING().notNull(),
+ STRING().notNull(),
+ BOOLEAN())
.testResult(
jsonString($("f0").parseJson()),
"JSON_STRING(PARSE_JSON(f0))",
@@ -1070,10 +1075,28 @@ class JsonFunctionsITCase extends
BuiltInFunctionTestBase {
jsonString($("f2").parseJson(true)),
"JSON_STRING(PARSE_JSON(f2, true))",
"{\"a\":2}",
- STRING().notNull()),
+ STRING().notNull())
+ // a NULL allowDuplicateKeys is treated as false
+ .testResult(
+ jsonString(call("PARSE_JSON", $("f0"),
$("f3"))),
+ "JSON_STRING(PARSE_JSON(f0, f3))",
+ "{\"a\":1,\"b\":[2,3]}",
+ STRING().notNull())
+ .testSqlRuntimeError(
+ "PARSE_JSON(f2, f3)",
+ TableRuntimeException.class,
+ "Failed to parse JSON string")
+ .testTableApiRuntimeError(
+ call("PARSE_JSON", $("f2"), $("f3")),
+ TableRuntimeException.class,
+ "Failed to parse JSON string"),
TestSetSpec.forFunction(BuiltInFunctionDefinitions.TRY_PARSE_JSON)
- .onFieldsWithData("{\"a\":1}", "1e400",
"{\"a\":1,\"a\":2}")
- .andDataTypes(STRING().notNull(), STRING().notNull(),
STRING().notNull())
+ .onFieldsWithData("{\"a\":1}", "1e400",
"{\"a\":1,\"a\":2}", null)
+ .andDataTypes(
+ STRING().notNull(),
+ STRING().notNull(),
+ STRING().notNull(),
+ BOOLEAN())
.testResult(
jsonString($("f0").tryParseJson()),
"JSON_STRING(TRY_PARSE_JSON(f0))",
@@ -1095,6 +1118,17 @@ class JsonFunctionsITCase extends
BuiltInFunctionTestBase {
jsonString($("f2").tryParseJson(true)),
"JSON_STRING(TRY_PARSE_JSON(f2, true))",
"{\"a\":2}",
+ STRING())
+ // a NULL allowDuplicateKeys is treated as false
+ .testResult(
+ jsonString(call("TRY_PARSE_JSON", $("f0"),
$("f3"))),
+ "JSON_STRING(TRY_PARSE_JSON(f0, f3))",
+ "{\"a\":1}",
+ STRING())
+ .testResult(
+ jsonString(call("TRY_PARSE_JSON", $("f2"),
$("f3"))),
+ "JSON_STRING(TRY_PARSE_JSON(f2, f3))",
+ null,
STRING()),
// Input is parsed from its UTF-8 bytes. A NUL or a BOM must
not be read as a
// UTF-16 or UTF-32 hint that turns invalid JSON into a value.
Only input that is
diff --git
a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/ParseJsonFunction.java
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/ParseJsonFunction.java
index 50f758e9b4d..7c2f5789c8f 100644
---
a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/ParseJsonFunction.java
+++
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/ParseJsonFunction.java
@@ -39,14 +39,17 @@ public class ParseJsonFunction extends
BuiltInScalarFunction {
return eval(jsonStr, false);
}
- public @Nullable Variant eval(@Nullable StringData jsonStr, boolean
allowDuplicateKeys) {
+ /** A {@code NULL} {@code allowDuplicateKeys} is treated as {@code false}.
*/
+ public @Nullable Variant eval(
+ @Nullable StringData jsonStr, @Nullable Boolean
allowDuplicateKeys) {
if (jsonStr == null) {
return null;
}
final byte[] bytes = jsonStr.toBytes();
try {
- return BinaryVariantInternalBuilder.parseJson(bytes,
allowDuplicateKeys);
+ return BinaryVariantInternalBuilder.parseJson(
+ bytes, Boolean.TRUE.equals(allowDuplicateKeys));
} catch (Throwable e) {
final int invalidIndex =
StringUtf8Utils.firstInvalidUtf8ByteIndex(bytes, 0,
bytes.length);
diff --git
a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/TryParseJsonFunction.java
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/TryParseJsonFunction.java
index 161e1c65131..c3e4dc66bef 100644
---
a/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/TryParseJsonFunction.java
+++
b/flink-table/flink-table-runtime/src/main/java/org/apache/flink/table/runtime/functions/scalar/TryParseJsonFunction.java
@@ -37,13 +37,16 @@ public class TryParseJsonFunction extends
BuiltInScalarFunction {
return eval(jsonStr, false);
}
- public @Nullable Variant eval(@Nullable StringData jsonStr, boolean
allowDuplicateKeys) {
+ /** A {@code NULL} {@code allowDuplicateKeys} is treated as {@code false}.
*/
+ public @Nullable Variant eval(
+ @Nullable StringData jsonStr, @Nullable Boolean
allowDuplicateKeys) {
if (jsonStr == null) {
return null;
}
try {
- return BinaryVariantInternalBuilder.parseJson(jsonStr.toBytes(),
allowDuplicateKeys);
+ return BinaryVariantInternalBuilder.parseJson(
+ jsonStr.toBytes(),
Boolean.TRUE.equals(allowDuplicateKeys));
} catch (Throwable e) {
return null;
}
