This is an automated email from the ASF dual-hosted git repository.
wernerdv pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push:
new 74d175bfcb6 IGNITE-29206 Serialize compressed and raw fields in
DirectMessageWriter through a fixed-size buffer (#13748)
74d175bfcb6 is described below
commit 74d175bfcb65a34a6d48da59e94f903bfc939526
Author: Dmitry Werner <[email protected]>
AuthorDate: Wed Oct 28 02:34:14 2026 +0800
IGNITE-29206 Serialize compressed and raw fields in DirectMessageWriter
through a fixed-size buffer (#13748)
---
.../internal/direct/DirectMessageWriter.java | 68 ++++++-----
.../managers/communication/CompressedMessage.java | 132 +++++++++++++--------
.../direct/DirectMarshallingMessagesTest.java | 32 +++++
.../communication/CompressedMessageTest.java | 103 ++++++++++++++--
4 files changed, 249 insertions(+), 86 deletions(-)
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java
b/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java
index 48c1af526fb..e57161d2638 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/direct/DirectMessageWriter.java
@@ -19,6 +19,7 @@ package org.apache.ignite.internal.direct;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.BitSet;
import java.util.Collection;
import java.util.List;
@@ -38,6 +39,7 @@ import org.apache.ignite.internal.util.GridLongList;
import org.apache.ignite.internal.util.tostring.GridToStringInclude;
import org.apache.ignite.internal.util.typedef.F;
import org.apache.ignite.internal.util.typedef.internal.S;
+import org.apache.ignite.internal.util.typedef.internal.U;
import org.apache.ignite.lang.IgniteOutClosure;
import org.apache.ignite.lang.IgniteProductVersion;
import org.apache.ignite.lang.IgniteUuid;
@@ -59,6 +61,9 @@ public class DirectMessageWriter implements MessageWriter {
/** Temporary buffer capacity. */
private static final int TMP_BUF_CAPACITY = 10 * 1024;
+ /** Empty array of byte arrays. */
+ private static final byte[][] EMPTY_BYTE_ARRAYS = new byte[0][];
+
/** State. */
@GridToStringInclude
private final DirectMessageState<StateItem> state;
@@ -81,7 +86,7 @@ public class DirectMessageWriter implements MessageWriter {
/** Reusable writer for compressed payload serialization. Lazily created;
thread-confined like the writer itself. */
private DirectMessageWriter tmpWriter;
- /** Reusable scratch buffer for compressed payload serialization. Retained
at the largest size seen so far. */
+ /** Reusable fixed-size scratch buffer for compressed payload
serialization. */
private ByteBuffer tmpBuf;
/** @param msgFactory Message factory. */
@@ -507,12 +512,12 @@ public class DirectMessageWriter implements MessageWriter
{
}
if (!stream.serializeFinished()) {
- // CompressedMessage consumes the scratch buffer in its
constructor (deflates into its own byte array),
- // so the buffer never escapes this method and can be reused
across fields; heap is cheaper than direct
- // here (no native alloc / Cleaner churn).
- ByteBuffer buf = serializeToTemporaryBuffer(consumer);
+ try (CompressedMessage.Compressor compressor = new
CompressedMessage.Compressor(compressionLvl)) {
+ serializeToTemporaryBuffer(consumer, compressor::compress);
+
+ stream.compressedMessage(compressor.message());
+ }
- stream.compressedMessage(new CompressedMessage(buf,
compressionLvl));
stream.serializeFinished(true);
}
@@ -526,11 +531,12 @@ public class DirectMessageWriter implements MessageWriter
{
/** {@inheritDoc} */
@Override public void addPendingRawField(int tag, Consumer<MessageWriter>
valueWriter) {
- ByteBuffer buf = serializeToTemporaryBuffer(valueWriter::accept);
+ List<byte[]> pieces = new ArrayList<>(1);
- byte[] bytes = new byte[buf.remaining()];
+ serializeToTemporaryBuffer(valueWriter::accept, piece ->
+ pieces.add(Arrays.copyOfRange(piece.array(), piece.position(),
piece.limit())));
- buf.get(bytes);
+ byte[] bytes = pieces.size() == 1 ? pieces.get(0) :
U.join(pieces.toArray(EMPTY_BYTE_ARRAYS));
state.item().pendingRawFields().add(new RawField(tag, bytes));
}
@@ -544,12 +550,14 @@ public class DirectMessageWriter implements MessageWriter
{
return stream.lastFinished();
}
- /** */
- private ByteBuffer
serializeToTemporaryBuffer(Consumer<DirectMessageWriter> consumer) {
+ /**
+ * Serializes through the temporary buffer as through a socket buffer:
each time the buffer fills up or the
+ * serialization finishes, its content is passed to the piece consumer and
the buffer is reused.
+ */
+ private void serializeToTemporaryBuffer(Consumer<DirectMessageWriter>
consumer, Consumer<ByteBuffer> pieceConsumer) {
+ // Heap is cheaper than direct here (no native alloc / Cleaner churn).
if (tmpBuf == null)
tmpBuf = ByteBuffer.allocate(TMP_BUF_CAPACITY);
- else
- tmpBuf.clear();
// Reuse the temp writer across fields/messages instead of allocating
a fresh state stack each time.
if (tmpWriter == null)
@@ -559,29 +567,33 @@ public class DirectMessageWriter implements MessageWriter
{
tmpWriter.setBuffer(tmpBuf);
- boolean finished;
+ boolean ok = false;
- do {
- if (tmpBuf.remaining() <= tmpBuf.capacity() / 10) {
- ByteBuffer newBuf = ByteBuffer.allocate(tmpBuf.capacity() * 2);
+ try {
+ boolean finished;
- tmpBuf.flip();
- newBuf.put(tmpBuf);
+ do {
+ tmpBuf.clear();
- tmpBuf = newBuf;
+ consumer.accept(tmpWriter);
- tmpWriter.setBuffer(tmpBuf);
- }
+ finished = tmpWriter.state.item().stream.lastFinished();
- consumer.accept(tmpWriter);
+ assert finished || tmpBuf.position() > 0 : "No progress in the
temporary buffer";
- finished = tmpWriter.state.item().stream.lastFinished();
- }
- while (!finished);
+ tmpBuf.flip();
- tmpBuf.flip();
+ pieceConsumer.accept(tmpBuf);
+ }
+ while (!finished);
- return tmpBuf;
+ ok = true;
+ }
+ finally {
+ // Don't reuse a writer with half-written state.
+ if (!ok)
+ tmpWriter = null;
+ }
}
/**
diff --git
a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/CompressedMessage.java
b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/CompressedMessage.java
index 0988452f6a5..3c99af130f7 100644
---
a/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/CompressedMessage.java
+++
b/modules/core/src/main/java/org/apache/ignite/internal/managers/communication/CompressedMessage.java
@@ -50,7 +50,7 @@ public class CompressedMessage extends AbstractMessage
implements NonMarshallabl
*/
private static final int MAX_DEFLATE_BLOCK_OVERHEAD = 64;
- /** Compressed data chunks: filled by {@link #compress(ByteBuffer)} on
send, by the serializer on receive. */
+ /** Compressed data chunks: filled by {@link Compressor} on send, by the
serializer on receive. */
List<byte[]> chunks;
/** Index of the next chunk to send. */
@@ -65,26 +65,11 @@ public class CompressedMessage extends AbstractMessage
implements NonMarshallabl
/** Flag indicating whether this is the last chunk. */
boolean finalChunk;
- /** Compression level. */
- private int compressionLvl;
-
/** Constructor. */
public CompressedMessage() {
// No-op.
}
- /**
- * @param buf Source buffer with serialized data.
- * @param compressionLvl Compression level.
- */
- public CompressedMessage(ByteBuffer buf, int compressionLvl) {
- dataSize = buf.remaining();
- this.compressionLvl = compressionLvl;
-
- if (dataSize > 0)
- compress(buf);
- }
-
/** @return Raw data size. */
public int dataSize() {
return dataSize;
@@ -102,40 +87,6 @@ public class CompressedMessage extends AbstractMessage
implements NonMarshallabl
return chunkIdx < chunks.size() ? chunks.get(chunkIdx++) : null;
}
- /** @param buf Buffer. */
- private void compress(ByteBuffer buf) {
- Deflater deflater = new Deflater(compressionLvl, true);
-
- try {
- deflater.setInput(buf);
- deflater.finish();
-
- chunks = new ArrayList<>(dataSize / CHUNK_SIZE + 1);
-
- // Incompressible data may expand: raw-deflate worst case below
CHUNK_SIZE is one stored block (~11 bytes overhead).
- byte[] chunk0 = new byte[dataSize >= CHUNK_SIZE ? CHUNK_SIZE :
dataSize + 16];
-
- int len = 0;
-
- while (!deflater.finished()) {
- len += deflater.deflate(chunk0, len, chunk0.length - len);
-
- if (len == chunk0.length && !deflater.finished()) {
- chunks.add(chunk0);
-
- chunk0 = new byte[CHUNK_SIZE];
- len = 0;
- }
- }
-
- if (len > 0)
- chunks.add(len == chunk0.length ? chunk0 :
Arrays.copyOf(chunk0, len));
- }
- finally {
- deflater.end();
- }
- }
-
/** @return Uncompressed data. */
private byte[] uncompress() {
if (chunks == null)
@@ -201,4 +152,85 @@ public class CompressedMessage extends AbstractMessage
implements NonMarshallabl
@Override public String toString() {
return S.toString(CompressedMessage.class, this);
}
+
+ /** Compresses serialized data piece by piece, so that the data never has
to be held in one buffer. */
+ public static class Compressor implements AutoCloseable {
+ /** */
+ private final Deflater deflater;
+
+ /** */
+ private final List<byte[]> chunks = new ArrayList<>();
+
+ /** Chunk being filled. */
+ private byte[] chunk;
+
+ /** Number of bytes in {@link #chunk}. */
+ private int chunkLen;
+
+ /** Raw data size. */
+ private int dataSize;
+
+ /** @param compressionLvl Compression level. */
+ public Compressor(int compressionLvl) {
+ deflater = new Deflater(compressionLvl, true);
+ }
+
+ /**
+ * Compresses the next piece of serialized data. The piece is consumed
fully, so its buffer can be reused for
+ * the next piece.
+ *
+ * @param piece Piece of serialized data.
+ */
+ public void compress(ByteBuffer piece) {
+ // Incompressible data may expand: raw-deflate worst case below
CHUNK_SIZE is one stored block (~11 bytes overhead).
+ if (chunk == null)
+ chunk = new byte[Math.min(CHUNK_SIZE, piece.remaining() + 16)];
+
+ dataSize += piece.remaining();
+
+ deflater.setInput(piece);
+
+ deflate(false);
+ }
+
+ /** @return Message with the compressed data. */
+ public CompressedMessage message() {
+ CompressedMessage msg = new CompressedMessage();
+
+ msg.dataSize = dataSize;
+
+ if (dataSize > 0) {
+ deflater.finish();
+
+ deflate(true);
+
+ if (chunkLen > 0)
+ chunks.add(chunkLen == chunk.length ? chunk :
Arrays.copyOf(chunk, chunkLen));
+
+ msg.chunks = chunks;
+ }
+
+ return msg;
+ }
+
+ /** @param finish Whether to deflate until the stream is finished
rather than until the input is consumed. */
+ private void deflate(boolean finish) {
+ do {
+ if (chunkLen == chunk.length) {
+ chunks.add(chunk);
+
+ chunk = new byte[CHUNK_SIZE];
+ chunkLen = 0;
+ }
+
+ chunkLen += deflater.deflate(chunk, chunkLen, chunk.length -
chunkLen);
+ }
+ while (finish ? !deflater.finished() : !deflater.needsInput());
+ }
+
+ /** {@inheritDoc} */
+ @Override public void close() {
+ deflater.end();
+ }
+ }
}
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/direct/DirectMarshallingMessagesTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/direct/DirectMarshallingMessagesTest.java
index 5eab512eb6e..a2eb82da641 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/direct/DirectMarshallingMessagesTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/direct/DirectMarshallingMessagesTest.java
@@ -22,11 +22,14 @@ import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
import java.util.Map;
+import java.util.Random;
import org.apache.ignite.internal.CoreMessagesProvider;
import
org.apache.ignite.internal.managers.communication.IgniteMessageFactoryImpl;
import org.apache.ignite.plugin.extensions.communication.Message;
import org.apache.ignite.plugin.extensions.communication.MessageFactory;
import
org.apache.ignite.plugin.extensions.communication.MessageFactoryProvider;
+import org.apache.ignite.plugin.extensions.communication.MessageReader;
+import org.apache.ignite.plugin.extensions.communication.RawField;
import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest;
import org.apache.ignite.transactions.TransactionIsolation;
import org.junit.Test;
@@ -123,6 +126,35 @@ public class DirectMarshallingMessagesTest extends
GridCommonAbstractTest {
assertNull(resMsg.isolationsList);
}
+ /** Raw field larger than the writer's temporary buffer is serialized in
several pieces. */
+ @Test
+ public void testLargeRawField() {
+ byte[] val = new byte[40_000];
+
+ new Random(42).nextBytes(val);
+
+ DirectMessageWriter writer = new DirectMessageWriter(msgFactory);
+
+ ByteBuffer buf = ByteBuffer.allocate(1 << 16);
+
+ writer.setBuffer(buf);
+
+ writer.addPendingRawField(1, w -> w.writeByteArray(val));
+
+ assertTrue(writer.writeRawFields(null));
+
+ buf.flip();
+
+ DirectMessageReader reader = new DirectMessageReader(msgFactory, null);
+
+ reader.setBuffer(buf);
+
+ RawField f = reader.readRawFields().get(0);
+
+ assertEquals(1, f.tag());
+ assertArrayEquals(val, reader.deserializeRawField(f,
MessageReader::readByteArray));
+ }
+
/**
* @param srcMsg Message to marshal.
* @param <T> Message type.
diff --git
a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/CompressedMessageTest.java
b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/CompressedMessageTest.java
index 2f9c77e4299..c1e259bd8e0 100644
---
a/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/CompressedMessageTest.java
+++
b/modules/core/src/test/java/org/apache/ignite/internal/managers/communication/CompressedMessageTest.java
@@ -61,7 +61,7 @@ public class CompressedMessageTest {
writer.setBuffer(tmpBuf);
- GridDhtPartitionsFullMessage fullMsg = fullMessage();
+ GridDhtPartitionsFullMessage fullMsg = fullMessage(500);
boolean finished = false;
boolean checkChunkCnt = true;
@@ -178,7 +178,7 @@ public class CompressedMessageTest {
public void testUncompressFailsOnUnderstatedDataSize() {
byte[] data = new byte[1000];
- CompressedMessage sent = new CompressedMessage(ByteBuffer.wrap(data),
Deflater.BEST_SPEED);
+ CompressedMessage sent = compress(ByteBuffer.wrap(data));
CompressedMessage rcvd = new CompressedMessage();
@@ -196,7 +196,7 @@ public class CompressedMessageTest {
new Random(42).nextBytes(data);
- CompressedMessage sent = new CompressedMessage(ByteBuffer.wrap(data),
Deflater.BEST_SPEED);
+ CompressedMessage sent = compress(ByteBuffer.wrap(data));
assertTrue(sent.chunks.size() > 1);
@@ -218,13 +218,13 @@ public class CompressedMessageTest {
writer.setBuffer(tmpBuf);
- assertTrue(writer.writeMessage(fullMessage(), false, LATEST_SCHEMA));
+ assertTrue(writer.writeMessage(fullMessage(500), false,
LATEST_SCHEMA));
tmpBuf.flip();
tmpBuf.limit(tmpBuf.limit() - 5); // Truncate the serialized message.
- CompressedMessage compressedMsg = new CompressedMessage(tmpBuf,
Deflater.BEST_SPEED);
+ CompressedMessage compressedMsg = compress(tmpBuf);
DirectMessageWriter wireWriter = new DirectMessageWriter(MSG_FACTORY);
@@ -243,12 +243,99 @@ public class CompressedMessageTest {
GridTestUtils.assertThrows(null, () -> reader.readMessage(true,
LATEST_SCHEMA), IgniteException.class, "ended unexpectedly");
}
- /** */
- private GridDhtPartitionsFullMessage fullMessage() {
+ /** Temporary buffer of the writer must not grow for a large compressed
field. */
+ @Test
+ public void testTemporaryBufferDoesNotGrow() {
+ DirectMessageWriter writer = new DirectMessageWriter(MSG_FACTORY);
+
+ writeAndRead(writer, fullMessage(10));
+
+ int tmpBufCap = U.<ByteBuffer>field(writer, "tmpBuf").capacity();
+
+ // Compressed size above the buffer capacity means the raw data was
serialized in several pieces.
+ assertTrue(writeAndRead(writer, fullMessage(10_000)) > tmpBufCap);
+
+ assertEquals(tmpBufCap, U.<ByteBuffer>field(writer,
"tmpBuf").capacity());
+ }
+
+ /** Writer must stay usable if serialization of a compressed field fails.
*/
+ @Test
+ public void testWriterReusableAfterFailedCompressedSerialization() {
+ Map<GroupPartitionIdPair, Long> brokenSuppliers = new HashMap<>();
+
+ // Null value fails with NPE in the middle of the nested map
serialization.
+ brokenSuppliers.put(new GroupPartitionIdPair(1, 2), null);
+
+ GridDhtPartitionsFullMessage brokenMsg = new
GridDhtPartitionsFullMessage(null, null, new AffinityTopologyVersion(0),
+ Map.of(UUID.randomUUID(), brokenSuppliers), Map.of());
+
+ DirectMessageWriter writer = new DirectMessageWriter(MSG_FACTORY);
+
+ writer.setBuffer(ByteBuffer.allocate(4096));
+
+ GridTestUtils.assertThrows(null, () -> writer.writeMessage(brokenMsg,
true, LATEST_SCHEMA), NullPointerException.class, null);
+
+ writeAndRead(writer, fullMessage(500));
+ }
+
+ /**
+ * Writes the message with compression through the given writer and checks
it is read back unchanged.
+ *
+ * @return Size of the written message.
+ */
+ private int writeAndRead(DirectMessageWriter writer,
GridDhtPartitionsFullMessage msg) {
+ writer.reset();
+
+ ByteBuffer buf = ByteBuffer.allocate(4096);
+ ByteBuffer msgBuf = ByteBuffer.allocate(1 << 20);
+
+ writer.setBuffer(buf);
+
+ boolean finished = false;
+
+ while (!finished) {
+ finished = writer.writeMessage(msg, true, LATEST_SCHEMA);
+
+ buf.flip();
+
+ msgBuf.put(buf);
+
+ buf.clear();
+ }
+
+ msgBuf.flip();
+
+ DirectMessageReader reader = new DirectMessageReader(MSG_FACTORY,
null);
+
+ reader.setBuffer(msgBuf);
+
+ Message readMsg = reader.readMessage(true, LATEST_SCHEMA);
+
+ assertTrue(readMsg instanceof GridDhtPartitionsFullMessage);
+
+ assertEqualsFullMsg(msg, (GridDhtPartitionsFullMessage)readMsg);
+
+ return msgBuf.limit();
+ }
+
+ /** @return Message with the compressed buffer content. */
+ private static CompressedMessage compress(ByteBuffer buf) {
+ try (CompressedMessage.Compressor compressor = new
CompressedMessage.Compressor(Deflater.BEST_SPEED)) {
+ compressor.compress(buf);
+
+ return compressor.message();
+ }
+ }
+
+ /**
+ * @param cnt Number of entries in the compressed maps.
+ * @return Full message.
+ */
+ private GridDhtPartitionsFullMessage fullMessage(int cnt) {
Map<UUID, Map<GroupPartitionIdPair, Long>> partHistSuppliers = new
HashMap<>();
Map<UUID, Map<Integer, Set<Integer>>> partsToReload = new HashMap<>();
- for (int i = 0; i < 500; i++) {
+ for (int i = 0; i < cnt; i++) {
UUID uuid = UUID.randomUUID();
partHistSuppliers.put(uuid, Map.of(new GroupPartitionIdPair(i, i +
1), i + 2L));