This is an automated email from the ASF dual-hosted git repository.
LuciferYang pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 8cc7d9a13693 [SPARK-57599][SQL] Decode Variant strings and keys as
UTF-8
8cc7d9a13693 is described below
commit 8cc7d9a13693eae79b3b1d1233eb20cdf2097d39
Author: YangJie <[email protected]>
AuthorDate: Thu Jun 25 20:59:22 2026 +0800
[SPARK-57599][SQL] Decode Variant strings and keys as UTF-8
### What changes were proposed in this pull request?
`VariantUtil.getString` and `VariantUtil.getMetadataKey` decoded their
bytes with `new String(bytes, offset, length)`, which uses the JVM default
charset. This PR decodes both with `StandardCharsets.UTF_8`, matching how
`VariantBuilder` writes keys and string values
(`getBytes(StandardCharsets.UTF_8)`) and the UTF-8 string data defined by the
variant binary format. A regression test, `VariantUtf8DecodeSuite`, is added in
the `common/variant` module next to `VariantUtil`.
### Why are the changes needed?
The variant binary stores dictionary keys and string content as UTF-8, but
the reader decoded them with the platform default charset, so the write and
read sides disagree whenever that charset is not UTF-8. On Java 17 the default
charset is environment-dependent (for example `US-ASCII` when the JVM starts
under `LANG=C`), and reading a variant that contains non-ASCII keys or strings
then silently corrupts those characters.
### Does this PR introduce _any_ user-facing change?
No change when the JVM default charset is UTF-8, which is the common case
and what CI runs. Under a non-UTF-8 default charset, operations that read
variant strings or keys -- `to_json(variant)`, casting a variant to string,
`schema_of_variant`, and field/key extraction -- now return the correct
characters instead of corrupting non-ASCII keys and string values.
### How was this patch tested?
Added `VariantUtf8DecodeSuite` in `common/variant`. A plain unit test
cannot exercise the bug, because the default charset is fixed at JVM startup
and pinned to `-Dfile.encoding=UTF-8` in CI, so the test forks a child JVM with
`-Dfile.encoding=ISO-8859-1` and round-trips a variant with non-ASCII object
keys and string values there, including a value over 63 UTF-8 bytes to cover
the `LONG_STR` path. With the pre-fix default-charset decode the characters are
corrupted (for example `café [...]
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude Code (Claude Opus 4.8)
Closes #56659 from LuciferYang/SPARK-variant-utf8-string-decode.
Authored-by: YangJie <[email protected]>
Signed-off-by: yangjie01 <[email protected]>
(cherry picked from commit 09205bbdeb9583a89dc868c6df98ce4295a4b97e)
Signed-off-by: yangjie01 <[email protected]>
---
.../apache/spark/types/variant/VariantUtil.java | 10 +-
.../types/variant/VariantUtf8DecodeSuite.scala | 118 +++++++++++++++++++++
2 files changed, 126 insertions(+), 2 deletions(-)
diff --git
a/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
b/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
index 795d46ec2062..e50ec4697300 100644
---
a/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
+++
b/common/variant/src/main/java/org/apache/spark/types/variant/VariantUtil.java
@@ -26,6 +26,7 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.UUID;
@@ -506,7 +507,10 @@ public class VariantUtil {
length = readUnsigned(value, pos + 1, U32_SIZE);
}
checkIndex(start + length - 1, value.length);
- return new String(value, start, length);
+ // The string content is UTF-8 encoded (it is written by
`VariantBuilder.appendString`).
+ // Decode with UTF-8 explicitly rather than relying on the JVM default
charset, which is
+ // platform-dependent on Java 17.
+ return new String(value, start, length, StandardCharsets.UTF_8);
}
throw unexpectedType(Type.STRING);
}
@@ -606,6 +610,8 @@ public class VariantUtil {
int nextOffset = readUnsigned(metadata, 1 + (id + 2) * offsetSize,
offsetSize);
if (offset > nextOffset) throw malformedVariant();
checkIndex(stringStart + nextOffset - 1, metadata.length);
- return new String(metadata, stringStart + offset, nextOffset - offset);
+ // Dictionary keys are UTF-8 encoded (see `VariantBuilder.addKey`). Decode
with UTF-8
+ // explicitly rather than relying on the platform-dependent JVM default
charset.
+ return new String(metadata, stringStart + offset, nextOffset - offset,
StandardCharsets.UTF_8);
}
}
diff --git
a/common/variant/src/test/scala/org/apache/spark/types/variant/VariantUtf8DecodeSuite.scala
b/common/variant/src/test/scala/org/apache/spark/types/variant/VariantUtf8DecodeSuite.scala
new file mode 100644
index 000000000000..e196209c82fa
--- /dev/null
+++
b/common/variant/src/test/scala/org/apache/spark/types/variant/VariantUtf8DecodeSuite.scala
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.types.variant
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Paths
+
+import org.scalatest.funsuite.AnyFunSuite // scalastyle:ignore funsuite
+
+class VariantUtf8DecodeSuite extends AnyFunSuite { // scalastyle:ignore
funsuite
+
+ test("SPARK-57599: keys and string values decode as UTF-8 under a non-UTF-8
default charset") {
+ // The reader must decode object keys and string values as UTF-8
regardless of the JVM default
+ // charset. That can only be exercised by changing the default charset,
which is fixed at JVM
+ // startup and pinned to UTF-8 in the test JVM, so fork a child JVM with a
non-UTF-8
+ // -Dfile.encoding and round-trip a variant with non-ASCII content there
(see
+ // VariantUtf8DecodeChecker). With the pre-fix default-charset decode the
characters are
+ // corrupted and the child exits non-zero.
+ val javaExe = Paths.get(sys.props("java.home"), "bin", "java").toString
+ val command = Seq(
+ javaExe,
+ "-Dfile.encoding=ISO-8859-1",
+ "-cp", sys.props("java.class.path"),
+ VariantUtf8DecodeChecker.getClass.getName.stripSuffix("$"))
+ val process = new ProcessBuilder(command:
_*).redirectErrorStream(true).start()
+ val output = new String(process.getInputStream.readAllBytes(),
StandardCharsets.UTF_8).trim
+ val exitCode = process.waitFor()
+ if (output.contains("RESULT=INCONCLUSIVE")) {
+ // The child JVM ended up with a UTF-8 default charset, so it cannot
exercise the fix.
+ assume(false, s"Inconclusive; child output: $output")
+ }
+ assert(exitCode === 0, s"child JVM reported a decode failure; output:
$output")
+ }
+}
+
+/**
+ * Entry point run in a child JVM by `VariantUtf8DecodeSuite` to verify that
the Variant reader
+ * decodes object keys and string values as UTF-8 independently of the JVM
default charset.
+ *
+ * This has to run as a separate `main`: the only way to actually exercise the
bug is to start a JVM
+ * with a non-UTF-8 `-Dfile.encoding`, because the default charset is fixed at
JVM startup and is
+ * pinned to UTF-8 in the test JVM. With the pre-fix default-charset decode,
the non-ASCII content
+ * round-tripped here is corrupted and the process exits non-zero.
+ *
+ * All output is ASCII (characters are reported as code points) so it is
readable regardless of the
+ * child's default charset. The process prints `RESULT=OK` and exits 0 on
success, `RESULT=FAIL` and
+ * exits 1 if any value was corrupted, or `RESULT=INCONCLUSIVE` and exits 0 if
the child's default
+ * charset turned out to be UTF-8 (in which case the fixed and buggy code
cannot be told apart).
+ */
+private[variant] object VariantUtf8DecodeChecker {
+
+ // This object is a standalone process whose stdout is its result protocol,
so println is used
+ // deliberately as the communication channel with the parent test.
+ // scalastyle:off println
+
+ // scalastyle:off nonascii
+ // (key, value) pairs covering 2-byte, 3-byte, and LONG_STR (> 63 UTF-8
bytes) UTF-8 content.
+ private val cases: Seq[(String, String)] = Seq(
+ "café" -> "résumé",
+ "你好" -> "世界",
+ "é" -> ("你" * 22)) // value is 66 UTF-8 bytes, over MAX_SHORT_STR_SIZE ->
LONG_STR encoding
+ // scalastyle:on nonascii
+
+ def main(args: Array[String]): Unit = {
+ // Read the startup `file.encoding`, which determines the JVM default
charset that the buggy
+ // `new String(bytes)` decode used. (Querying the default charset directly
is banned by
+ // scalastyle.)
+ val fileEncoding = sys.props.getOrElse("file.encoding", "")
+ if (fileEncoding.equalsIgnoreCase("UTF-8")) {
+ // The fixed and buggy code are indistinguishable when the default
charset is already UTF-8.
+ println(s"RESULT=INCONCLUSIVE fileEncoding=$fileEncoding")
+ System.exit(0)
+ }
+ var failures = 0
+ cases.foreach { case (key, value) =>
+ val json = "{" + jsonString(key) + ":" + jsonString(value) + "}"
+ val field = VariantBuilder.parseJson(json, false).getFieldAtIndex(0)
+ if (field.key != key) {
+ println(s"KEY_MISMATCH expected=${codePoints(key)}
actual=${codePoints(field.key)}")
+ failures += 1
+ }
+ val decoded = field.value.getString
+ if (decoded != value) {
+ println(s"VALUE_MISMATCH expected=${codePoints(value)}
actual=${codePoints(decoded)}")
+ failures += 1
+ }
+ }
+ if (failures == 0) {
+ println(s"RESULT=OK fileEncoding=$fileEncoding")
+ System.exit(0)
+ } else {
+ println(s"RESULT=FAIL failures=$failures fileEncoding=$fileEncoding")
+ System.exit(1)
+ }
+ }
+
+ private def jsonString(s: String): String = "\"" + s + "\""
+
+ private def codePoints(s: String): String =
+ s.codePoints().toArray.map(c => "U+%04X".format(c)).mkString("[", ",", "]")
+
+ // scalastyle:on println
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
