This is an automated email from the ASF dual-hosted git repository.
nizhikov pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new a9bc0d673ae IGNITE-27088 BinaryWriter should use internal String#value
(#13529)
a9bc0d673ae is described below
commit a9bc0d673aecd559fe1f881bc8000e36c7bbac93
Author: Nikolay <[email protected]>
AuthorDate: Fri Sep 11 18:44:13 2026 +0300
IGNITE-27088 BinaryWriter should use internal String#value (#13529)
---
.../jmh/binary/JmhBinaryStringWriteBenchmark.java | 166 ++++++++++++
.../ignite/internal/binary/BinaryWriterExImpl.java | 25 +-
.../ignite/internal/binary/StringWriter.java | 281 +++++++++++++++++++++
.../ignite/IgniteCommonsSystemProperties.java | 10 +
.../direct/stream/DirectByteBufferStream.java | 12 +-
.../internal/binary/StringWriterSelfTest.java | 203 +++++++++++++++
.../testsuites/IgniteBinaryObjectsTestSuite.java | 2 +
7 files changed, 681 insertions(+), 18 deletions(-)
diff --git
a/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java
b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java
new file mode 100644
index 00000000000..a2a3beb48c2
--- /dev/null
+++
b/modules/benchmarks/src/main/java/org/apache/ignite/internal/benchmarks/jmh/binary/JmhBinaryStringWriteBenchmark.java
@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.benchmarks.jmh.binary;
+
+import org.apache.ignite.internal.benchmarks.jmh.runner.JmhIdeBenchmarkRunner;
+import org.apache.ignite.internal.binary.StringWriter;
+import org.apache.ignite.internal.binary.streams.BinaryOutputStream;
+import org.apache.ignite.internal.binary.streams.BinaryStreams;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+import org.openjdk.jmh.profile.GCProfiler;
+import org.openjdk.jmh.runner.Runner;
+import org.openjdk.jmh.runner.options.OptionsBuilder;
+
+import static java.util.concurrent.TimeUnit.NANOSECONDS;
+import static java.util.concurrent.TimeUnit.SECONDS;
+import static
org.apache.ignite.IgniteCommonsSystemProperties.IGNITE_BINARY_STRING_ZERO_COPY;
+import static org.openjdk.jmh.annotations.Mode.AverageTime;
+import static org.openjdk.jmh.annotations.Scope.Thread;
+
+/**
+ * Compares zero-copy string serialization with the legacy serialization.
+ * @see
org.apache.ignite.IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY
+ */
+@State(Thread)
+@OutputTimeUnit(NANOSECONDS)
+@BenchmarkMode(AverageTime)
+@Warmup(iterations = 5, time = 5, timeUnit = SECONDS)
+@Measurement(iterations = 5, time = 10, timeUnit = SECONDS)
+public class JmhBinaryStringWriteBenchmark {
+ /** */
+ @Param({"true", "false"})
+ private boolean zeroCopy;
+
+ /** */
+ @Param({"8", "64", "512", "4096"})
+ private int len;
+
+ /** */
+ @Param({"ascii", "latin1", "cyrillic", "mixed"})
+ private String content;
+
+ /** */
+ private BinaryOutputStream out;
+
+ /** */
+ private String str;
+
+ /** */
+ public static void main(String[] args) throws Exception {
+ OptionsBuilder builder = JmhIdeBenchmarkRunner.create()
+ .forks(1)
+ .benchmarks(JmhBinaryStringWriteBenchmark.class.getName())
+ .profilers(GCProfiler.class)
+ .optionsBuilder();
+
+ new Runner(builder.build()).run();
+ }
+
+ /** */
+ @Setup
+ public void setup() {
+ // Must be set before the first use of StringWriter in this JVM.
+ System.setProperty(IGNITE_BINARY_STRING_ZERO_COPY,
String.valueOf(zeroCopy));
+
+ StringBuilder sb = new StringBuilder(len);
+
+ for (int i = 0; sb.length() < len; i++) {
+ switch (content) {
+ case "ascii":
+ sb.append((char)('a' + i % 26));
+
+ break;
+
+ case "latin1":
+ // Every 8th char is a Latin-1 char with the sign bit set.
+ sb.append(i % 8 == 7 ? (char)(0xC0 + i % 0x20) :
(char)('a' + i % 26));
+
+ break;
+
+ case "cyrillic":
+ sb.append((char)('\u0410' + i % 32));
+
+ break;
+
+ case "mixed":
+ // ASCII, Latin-1, 2-byte, 3-byte chars and a surrogate
pair.
+ switch (i % 5) {
+ case 0:
+ sb.append((char)('a' + i % 26));
+
+ break;
+
+ case 1:
+ sb.append('\u00e9');
+
+ break;
+
+ case 2:
+ sb.append('\u0416');
+
+ break;
+
+ case 3:
+ sb.append('\u20ac');
+
+ break;
+
+ default:
+ sb.append("\ud83d\ude00");
+ }
+
+ break;
+
+ default:
+ throw new IllegalArgumentException("Unknown content type:
" + content);
+ }
+ }
+
+ str = sb.toString();
+
+ out = BinaryStreams.outputStream(4 * len + 64);
+ }
+
+ /** */
+ @TearDown
+ public void tearDown() {
+ out.close();
+ }
+
+ /** */
+ @Benchmark
+ public void writeString(Blackhole bh) {
+ out.position(0);
+
+ if (zeroCopy)
+ StringWriter.write(str, out);
+ else
+ StringWriter.writeStringLegacy(str, out);
+
+ bh.consume(out.position());
+ }
+}
diff --git
a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
index 176a1ec9eb1..d2eded81638 100644
---
a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
+++
b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/BinaryWriterExImpl.java
@@ -30,6 +30,7 @@ import java.util.Date;
import java.util.Map;
import java.util.UUID;
import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteCommonsSystemProperties;
import org.apache.ignite.binary.BinaryObjectException;
import org.apache.ignite.binary.BinaryRawWriter;
import org.apache.ignite.internal.UnregisteredClassException;
@@ -40,13 +41,17 @@ import org.apache.ignite.internal.util.typedef.internal.A;
import org.apache.ignite.marshaller.Marshallers;
import org.jetbrains.annotations.Nullable;
-import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.ignite.IgniteCommonsSystemProperties.DFLT_ZERO_COPY;
+import static
org.apache.ignite.IgniteCommonsSystemProperties.IGNITE_BINARY_STRING_ZERO_COPY;
import static org.apache.ignite.internal.util.CommonUtils.MAX_ARRAY_SIZE;
/**
* Binary writer implementation.
*/
class BinaryWriterExImpl implements BinaryWriterEx {
+ /** Zero-copy serialization enabled flag. */
+ static final boolean ZERO_COPY =
IgniteCommonsSystemProperties.getBoolean(IGNITE_BINARY_STRING_ZERO_COPY,
DFLT_ZERO_COPY);
+
/** Length: integer. */
private static final int LEN_INT = 4;
@@ -733,20 +738,10 @@ class BinaryWriterExImpl implements BinaryWriterEx {
@Override public void writeString(@Nullable String val) throws
BinaryObjectException {
if (val == null)
out.writeByte(GridBinaryMarshaller.NULL);
- else {
- byte[] strArr;
-
- if (BinaryUtils.USE_STR_SERIALIZATION_VER_2)
- strArr = BinaryUtils.strToUtf8Bytes(val);
- else
- strArr = val.getBytes(UTF_8);
-
- out.unsafeEnsure(1 + 4);
- out.unsafeWriteByte(GridBinaryMarshaller.STRING);
- out.unsafeWriteInt(strArr.length);
-
- out.writeByteArray(strArr);
- }
+ else if (ZERO_COPY)
+ StringWriter.write(val, out);
+ else
+ StringWriter.writeStringLegacy(val, out);
}
/** {@inheritDoc} */
diff --git
a/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java
b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java
new file mode 100644
index 00000000000..36e77925cb2
--- /dev/null
+++
b/modules/binary/impl/src/main/java/org/apache/ignite/internal/binary/StringWriter.java
@@ -0,0 +1,281 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.binary;
+
+import java.lang.invoke.MethodHandle;
+import java.lang.invoke.MethodHandles;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
+import org.apache.ignite.IgniteCommonsSystemProperties;
+import org.apache.ignite.internal.binary.streams.BinaryOutputStream;
+import org.apache.ignite.internal.util.GridUnsafe;
+import org.apache.ignite.lang.IgniteBiTuple;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.ignite.internal.binary.BinaryWriterExImpl.ZERO_COPY;
+
+/**
+ * Writes {@link String} values to a {@link BinaryOutputStream} in UTF-8
without allocation of temporary byte arrays.
+ *
+ * @see IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY
+ */
+public final class StringWriter {
+ /** Latin-1 value of the {@code java.lang.String#coder} field. */
+ private static final byte LATIN1 = 0;
+
+ /** Offset of the {@code java.lang.String#value} field, or {@code -1} if
the compact string fast path is unavailable. */
+ private static final long STR_VALUE_OFF;
+
+ /** Offset of the {@code java.lang.String#coder} field, or {@code -1} if
the compact string fast path is unavailable. */
+ private static final long STR_CODER_OFF;
+
+ static {
+ IgniteBiTuple<Long, Long> result = fieldsOffsets();
+
+ STR_VALUE_OFF = result.get1();
+ STR_CODER_OFF = result.get2();
+ }
+
+ /**
+ * Handle of the intrinsified {@code java.lang.StringCoding#hasNegatives},
or {@code null} if unavailable.
+ * The intrinsic scans the array with SIMD instructions, far faster than
any scalar loop.
+ */
+ private static final MethodHandle HAS_NEGATIVES = hasNegatives();
+
+ /** */
+ private StringWriter() {
+ // No-op.
+ }
+
+ /**
+ * Writes a string to the output stream.
+ *
+ * @param val Value.
+ * @param out Output stream.
+ */
+ public static void write(@NotNull String val, BinaryOutputStream out) {
+ // 1 byte for `GridBinaryMarshaller.STRING` and integer (4 bytes) for
length.
+ out.unsafeEnsure(1 + 4);
+ out.unsafeWriteByte(GridBinaryMarshaller.STRING);
+
+ int lenPos = out.position();
+
+ out.unsafePosition(out.position() + 4);
+
+ int written;
+
+ byte[] latin1 = latin1Value(val);
+
+ if (latin1 != null) {
+ if (out.hasArray() && !hasNegatives(latin1)) {
+ out.unsafeEnsure(latin1.length);
+ // Pure ASCII: UTF-8 representation matches the internal
array, copy it as-is.
+ System.arraycopy(latin1, 0, out.array(), out.position(),
latin1.length);
+
+ written = latin1.length;
+
+ out.unsafePosition(out.position() + written);
+ }
+ else
+ written = writeLatin1(latin1, out);
+ }
+ else
+ written = writeChars(val, out);
+
+ out.unsafeWriteInt(lenPos, written);
+ }
+
+ /**
+ * Writes a Latin-1 encoded string value to the stream.
+ *
+ * @param val Internal Latin-1 array of the string.
+ * @param out Output stream.
+ * @return Number of bytes written.
+ */
+ private static int writeLatin1(byte[] val, BinaryOutputStream out) {
+ out.unsafeEnsure(Math.addExact(val.length, val.length));
+
+ int start = out.position();
+
+ for (int i = 0; i < val.length; i++) {
+ byte b = val[i];
+
+ if (b >= 0)
+ out.unsafeWriteByte(b);
+ else {
+ int c = b & 0b1111_1111;
+
+ out.unsafeWriteByte((byte)(0b1100_0000 | (c >> 6)));
+ out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111)));
+ }
+ }
+
+ return out.position() - start;
+ }
+
+ /**
+ * Writes string chars UTF-8 encoded to the stream. Replicates {@code
String#getBytes(UTF_8)} behavior exactly,
+ * including replacement of malformed surrogates with {@code '?'}. Stream
capacity must be ensured by the caller.
+ *
+ * @param val Value.
+ * @param out Output stream.
+ * @return Number of bytes written.
+ */
+ private static int writeChars(String val, BinaryOutputStream out) {
+ // Allocating memory for worst case - 3 bytes per char.
+ out.unsafeEnsure(Math.multiplyExact(3, val.length()));
+
+ int start = out.position();
+ int len = val.length();
+
+ for (int i = 0; i < len; i++) {
+ char c = val.charAt(i);
+
+ if (c < 0x80)
+ out.unsafeWriteByte((byte)c);
+ else if (c < 0x800) {
+ out.unsafeWriteByte((byte)(0b11_000000 | (c >> 6)));
+ out.unsafeWriteByte((byte)(0b10_000000 | (c & 0b00_111111)));
+ }
+ else if (!Character.isSurrogate(c)) {
+ out.unsafeWriteByte((byte)(0b1110_0000 | (c >> 12)));
+ out.unsafeWriteByte((byte)(0b1000_0000 | ((c >> 6) &
0b0011_1111)));
+ out.unsafeWriteByte((byte)(0b1000_0000 | (c & 0b0011_1111)));
+ }
+ else {
+ char c2;
+
+ if (Character.isHighSurrogate(c) && i + 1 < len &&
Character.isLowSurrogate(c2 = val.charAt(i + 1))) {
+ int cp = Character.toCodePoint(c, c2);
+
+ out.unsafeWriteByte((byte)(0b1111_0000 | (cp >> 18)));
+ out.unsafeWriteByte((byte)(0b1000_0000 | ((cp >> 12) &
0b0011_1111)));
+ out.unsafeWriteByte((byte)(0b1000_0000 | ((cp >> 6) &
0b0011_1111)));
+ out.unsafeWriteByte((byte)(0b1000_0000 | (cp &
0b0011_1111)));
+
+ i++;
+ }
+ else
+ out.unsafeWriteByte((byte)'?');
+ }
+ }
+
+ return out.position() - start;
+ }
+
+ /**
+ * @param val String.
+ * @return Internal Latin-1 array of the string,
+ * or {@code null} if the string is UTF-16 encoded or the internal
layout of {@link String} is unknown.
+ */
+ @Nullable public static byte[] latin1Value(String val) {
+ if (STR_VALUE_OFF < 0 || GridUnsafe.getByteField(val, STR_CODER_OFF)
!= LATIN1)
+ return null;
+
+ return (byte[])GridUnsafe.getObjectField(val, STR_VALUE_OFF);
+ }
+
+ /**
+ * @param arr Array.
+ * @return {@code True} if the array contains a byte with the sign bit set.
+ */
+ public static boolean hasNegatives(byte[] arr) {
+ if (HAS_NEGATIVES != null) {
+ try {
+ return (boolean)HAS_NEGATIVES.invokeExact(arr, 0, arr.length);
+ }
+ catch (Throwable ignored) {
+ // Fall through to the generic implementation.
+ }
+ }
+
+ // 8-byte strides with an early exit.
+ int i = 0;
+
+ for (int lim = arr.length - Long.BYTES; i <= lim; i += Long.BYTES) {
+ long hasNegatives = GridUnsafe.getLong(arr,
GridUnsafe.BYTE_ARR_OFF + i)
+ &
0b10000000_10000000_10000000_10000000_10000000_10000000_10000000_10000000L;
+
+ if (hasNegatives != 0)
+ return true;
+ }
+
+ for (; i < arr.length; i++) {
+ if (arr[i] < 0)
+ return true;
+ }
+
+ return false;
+ }
+
+ /** */
+ private static IgniteBiTuple<Long, Long> fieldsOffsets() {
+ if (ZERO_COPY) {
+ try {
+ Field valField = String.class.getDeclaredField("value");
+ Field coderField = String.class.getDeclaredField("coder");
+
+ return new IgniteBiTuple<>(
+ GridUnsafe.objectFieldOffset(valField),
+ GridUnsafe.objectFieldOffset(coderField)
+ );
+ }
+ catch (Throwable ignored) {
+ // No-op.
+ }
+ }
+
+ return new IgniteBiTuple<>(-1L, -1L);
+ }
+
+ /** */
+ private static @Nullable MethodHandle hasNegatives() {
+ if (ZERO_COPY) {
+ try {
+ Method mtd =
Class.forName("java.lang.StringCoding").getDeclaredMethod("hasNegatives",
byte[].class, int.class, int.class);
+
+ mtd.setAccessible(true);
+
+ return MethodHandles.lookup().unreflect(mtd);
+ }
+ catch (Throwable ignored) {
+ // No-op.
+ }
+ }
+
+ return null;
+ }
+
+ /** */
+ public static void writeStringLegacy(@NotNull String val,
BinaryOutputStream out) {
+ byte[] strArr;
+
+ if (BinaryUtils.USE_STR_SERIALIZATION_VER_2)
+ strArr = BinaryUtils.strToUtf8Bytes(val);
+ else
+ strArr = val.getBytes(UTF_8);
+
+ out.unsafeEnsure(1 + 4);
+ out.unsafeWriteByte(GridBinaryMarshaller.STRING);
+ out.unsafeWriteInt(strArr.length);
+
+ out.writeByteArray(strArr);
+ }
+}
diff --git
a/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java
b/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java
index ec443398ce3..62e4b5f8318 100644
---
a/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java
+++
b/modules/commons/src/main/java/org/apache/ignite/IgniteCommonsSystemProperties.java
@@ -38,6 +38,9 @@ public class IgniteCommonsSystemProperties {
/** Default value of {@link
IgniteCommonsSystemProperties#IGNITE_USE_BINARY_ARRAYS}. */
public static final boolean DFLT_IGNITE_USE_BINARY_ARRAYS = false;
+ /** Default value of {@link
IgniteCommonsSystemProperties#IGNITE_BINARY_STRING_ZERO_COPY}. */
+ public static final boolean DFLT_ZERO_COPY = true;
+
/**
* Setting to {@code true} enables writing sensitive information in {@code
toString()} output.
*/
@@ -128,6 +131,13 @@ public class IgniteCommonsSystemProperties {
public static final String
IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2 =
"IGNITE_BINARY_MARSHALLER_USE_STRING_SERIALIZATION_VER_2";
+ /**
+ * Enables zero-copy UTF-8 serialization of {@link String} values.
+ * Default value is {@code true}.
+ */
+ @SystemProperty(value = "Enables zero-copy UTF-8 serialization of String
values", defaults = "" + DFLT_ZERO_COPY)
+ public static final String IGNITE_BINARY_STRING_ZERO_COPY =
"IGNITE_BINARY_STRING_ZERO_COPY";
+
/**
* Enables storage of typed arrays.
* The default value is {@code BinaryUtils#DFLT_IGNITE_USE_BINARY_ARRAYS}.
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java
b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java
index c0752343b54..d2176dfb6f6 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/direct/stream/DirectByteBufferStream.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.direct.stream;
import java.lang.reflect.Array;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.BitSet;
import java.util.Collection;
@@ -33,6 +34,7 @@ import java.util.function.BooleanSupplier;
import java.util.function.Supplier;
import org.apache.ignite.IgniteCheckedException;
import org.apache.ignite.IgniteException;
+import org.apache.ignite.internal.binary.StringWriter;
import org.apache.ignite.internal.managers.communication.CompressedMessage;
import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion;
import org.apache.ignite.internal.processors.cache.CacheObject;
@@ -733,8 +735,12 @@ public class DirectByteBufferStream {
*/
public void writeString(String val) {
if (val != null) {
- if (curStrBackingArr == null)
- curStrBackingArr = val.getBytes();
+ if (curStrBackingArr == null) {
+ curStrBackingArr = StringWriter.latin1Value(val);
+
+ if (curStrBackingArr == null ||
StringWriter.hasNegatives(curStrBackingArr))
+ curStrBackingArr = val.getBytes(StandardCharsets.UTF_8);
+ }
writeByteArray(curStrBackingArr);
@@ -1362,7 +1368,7 @@ public class DirectByteBufferStream {
public String readString() {
byte[] arr = readByteArray();
- return arr != null ? new String(arr) : null;
+ return arr != null ? new String(arr, StandardCharsets.UTF_8) : null;
}
/**
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java
new file mode 100644
index 00000000000..da8312895e1
--- /dev/null
+++
b/modules/core/src/test/java/org/apache/ignite/internal/binary/StringWriterSelfTest.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.binary;
+
+import java.util.Arrays;
+import java.util.concurrent.ThreadLocalRandom;
+import org.apache.ignite.internal.binary.streams.BinaryOutputStream;
+import org.apache.ignite.internal.binary.streams.BinaryStreams;
+import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
+import org.junit.Test;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * Tests that {@link StringWriter} output is byte-identical to serialization
of the {@link String#getBytes()} result,
+ * which was used before zero-copy string serialization was introduced.
+ */
+public class StringWriterSelfTest extends GridCommonAbstractTest {
+ /** */
+ public static final int ASCII_MAX = 0x80;
+
+ /** */
+ public static final int LATIN1_MAX = 0x100;
+
+ /** */
+ public static final int TWO_BYTES_MAX = 0x800;
+
+ /** */
+ public static final int CHAR_HIGH_BOUND = 0x10000;
+
+ /** Tests for all encoder paths: ASCII bulk copy, Latin-1, generic UTF-16
and malformed surrogates. */
+ @Test
+ public void testCorpus() {
+ String[] cases = {
+ "",
+ "a",
+ "?",
+ "abcdefghijklmnopqrstuvwxyz0123456789", // Long ASCII: exercises
the 8-byte stride scan and bulk copy.
+ "caf\u00e9", // Latin-1 with a negative
byte.
+ "\u00ff\u0080\u00a0", // Latin-1, negative bytes
only.
+ "\u041f\u0440\u0438\u0432\u0435\u0442", // Cyrillic: 2-byte UTF-8
sequences.
+ "\u0800\u1234\uffff", // 3-byte UTF-8 sequences.
+ "\ud83d\ude00", // Emoji: valid surrogate
pair.
+ "a\ud83d\ude00b\u00e9\u0416\u0001", // Mixed content.
+ "\ud800", // Lone high surrogate.
+ "\udc00", // Lone low surrogate.
+ "a\ud800", // High surrogate at the
end.
+ "\ud800a", // High surrogate followed
by a regular char.
+ "\ud800\ud800", // Two high surrogates.
+ "\udc00\ud800", // Low surrogate before a
high one.
+ "\u0000", // NUL char.
+ "nul\u0000nul"
+ };
+
+ for (String str : cases)
+ check(str);
+ }
+
+ /** Randomized differential test against {@link String#getBytes()}. */
+ @Test
+ public void testRandomStrings() {
+ ThreadLocalRandom rnd = ThreadLocalRandom.current();
+
+ for (int iter = 0; iter < 100; iter++) {
+ StringBuilder sb = new StringBuilder(1 + rnd.nextInt(42));
+
+ for (int i = 0; i < sb.capacity(); i++) {
+ int bucket = rnd.nextInt(100);
+
+ char c;
+
+ if (bucket < 40)
+ // ASCII.
+ c = (char)rnd.nextInt(ASCII_MAX);
+ else if (bucket < 55)
+ // Latin-1.
+ c = (char)(ASCII_MAX + rnd.nextInt(LATIN1_MAX -
ASCII_MAX));
+ else if (bucket < 65)
+ // Other 2-byte chars.
+ c = (char)(LATIN1_MAX + rnd.nextInt(TWO_BYTES_MAX -
LATIN1_MAX));
+ else if (bucket < 75)
+ // 3-byte chars.
+ c = (char)(TWO_BYTES_MAX +
rnd.nextInt(Character.MIN_SURROGATE - TWO_BYTES_MAX));
+ else if (bucket < 90)
+ // Surrogates, mostly malformed.
+ c = (char)(Character.MIN_SURROGATE +
rnd.nextInt((Character.MAX_SURROGATE + 1) - Character.MIN_SURROGATE));
+ else
+ // 3-byte chars above the surrogate range.
+ c = (char)((Character.MAX_SURROGATE + 1) +
rnd.nextInt(CHAR_HIGH_BOUND - (Character.MAX_SURROGATE + 1)));
+
+ sb.append(c);
+ }
+
+ assertFalse(sb.isEmpty());
+
+ check(sb.toString());
+ }
+ }
+
+ /** Tests strings whose UTF-8 form is larger than the stream's minimal
capacity. */
+ @Test
+ public void testLargeStrings() {
+ int len = 100_000;
+
+ StringBuilder ascii = new StringBuilder(len);
+ StringBuilder latin1 = new StringBuilder(len);
+ StringBuilder cyrillic = new StringBuilder(len);
+ StringBuilder mixed = new StringBuilder(len * 6);
+
+ for (int i = 0; i < len; i++) {
+ ascii.append((char)('a' + i % 26));
+ // Every char is a Latin-1 char with the sign bit set: worst case
for the 2-bytes-per-char reservation.
+ latin1.append((char)(ASCII_MAX + i % (LATIN1_MAX - ASCII_MAX)));
+ cyrillic.append((char)('\u0410' + i % 32));
+ mixed.append((char)('a' + i %
26)).append('\u00e9').append('\u0416').append('\u20ac').append("\ud83d\ude00");
+ }
+
+ check(ascii.toString());
+ check(latin1.toString());
+ check(cyrillic.toString());
+ check(mixed.toString());
+ }
+
+ /** Tests that the stream position is correct after a string write, so
surrounding values are not corrupted. */
+ @Test
+ public void testStreamPosition() {
+ int int1 = 0xDEADBEEF;
+ String str1 = "caf\u00e9";
+ String str2 = "\ud83d\ude00";
+ int int2 = 0xCAFEBABE;
+
+ // Small initial capacity to check buffer reallocation.
+ try (BinaryOutputStream out = BinaryStreams.outputStream(2)) {
+ out.writeInt(int1);
+ StringWriter.write(str1, out);
+ StringWriter.write(str2, out);
+ out.writeInt(int2);
+
+ byte[] strBytes1 = strBytes(str1);
+ byte[] strBytes2 = strBytes(str2);
+
+ byte[] exp = new byte[Integer.BYTES + strBytes1.length +
strBytes2.length + Integer.BYTES];
+
+ System.arraycopy(intBytes(int1), 0, exp, 0, Integer.BYTES);
+ System.arraycopy(strBytes1, 0, exp, Integer.BYTES,
strBytes1.length);
+ System.arraycopy(strBytes2, 0, exp, Integer.BYTES +
strBytes1.length, strBytes2.length);
+ System.arraycopy(intBytes(int2), 0, exp, Integer.BYTES +
strBytes1.length + strBytes2.length, Integer.BYTES);
+
+ assertTrue(Arrays.equals(exp, out.arrayCopy()));
+ }
+ }
+
+ /**
+ * Checks that serialized form of the given string is byte-identical to
serialization of the {@link String#getBytes()} result.
+ * @param str String to check.
+ */
+ private void check(String str) {
+ try (BinaryOutputStream out = BinaryStreams.outputStream(1)) {
+ StringWriter.write(str, out);
+
+ assertTrue("String serialization mismatch: " + str,
Arrays.equals(strBytes(str), out.arrayCopy()));
+ }
+ }
+
+ /**
+ * @param str String.
+ * @return Expected serialized form of the string: flag, UTF-8 length and
UTF-8 bytes.
+ */
+ private static byte[] strBytes(String str) {
+ byte[] bytes = str.getBytes(UTF_8);
+ byte[] res = new byte[Byte.BYTES + Integer.BYTES + bytes.length];
+
+ res[0] = GridBinaryMarshaller.STRING;
+
+ System.arraycopy(intBytes(bytes.length), 0, res, 1, 4);
+ System.arraycopy(bytes, 0, res, 5, bytes.length);
+
+ return res;
+ }
+
+ /**
+ * @param val Value.
+ * @return Little-endian representation of the value.
+ */
+ private static byte[] intBytes(int val) {
+ return new byte[] {(byte)val, (byte)(val >> 8), (byte)(val >> 16),
(byte)(val >> 24)};
+ }
+}
diff --git
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
index 69b639df6b1..bdc9230a585 100644
---
a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
+++
b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteBinaryObjectsTestSuite.java
@@ -46,6 +46,7 @@ import
org.apache.ignite.internal.binary.GridBinaryWildcardsSelfTest;
import
org.apache.ignite.internal.binary.GridDefaultBinaryMappersBinaryMetaDataSelfTest;
import
org.apache.ignite.internal.binary.GridSimpleLowerCaseBinaryMappersBinaryMetaDataSelfTest;
import org.apache.ignite.internal.binary.RawBinaryObjectExtractorTest;
+import org.apache.ignite.internal.binary.StringWriterSelfTest;
import
org.apache.ignite.internal.binary.builder.BinaryObjectBuilderAdditionalSelfTest;
import
org.apache.ignite.internal.binary.noncompact.BinaryFieldsHeapNonCompactSelfTest;
import
org.apache.ignite.internal.binary.noncompact.BinaryFieldsOffheapNonCompactSelfTest;
@@ -107,6 +108,7 @@ import org.junit.runners.Suite;
BinaryTreeSelfTest.class,
BinaryMarshallerSelfTest.class,
+ StringWriterSelfTest.class,
BinaryObjectExceptionSelfTest.class,
BinarySerialiedFieldComparatorSelfTest.class,
