This is an automated email from the ASF dual-hosted git repository.
pjfanning pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/pekko-http.git
The following commit(s) were added to refs/heads/main by this push:
new a63fd46c3 perf: hand ByteBuffers to Inflater and CRC32 in the coding
stages (#1228)
a63fd46c3 is described below
commit a63fd46c3b30b9e093c513a91e4f16443498c1cd
Author: PJ Fanning <[email protected]>
AuthorDate: Fri Sep 11 10:00:58 2026 +0100
perf: hand ByteBuffers to Inflater and CRC32 in the coding stages (#1228)
Motivation:
Three places in the gzip/deflate stages copy a ByteString into a byte array
only
to hand it to java.util.zip:
- DeflateDecompressorBase.Inflate.parse calls setInput with the whole
remaining
buffer on every parse round, via toArray, so every round copies it.
- GzipCompressor.updateCrc and GzipDecompressor.crc16 use toArrayUnsafe,
which
only avoids the copy for a compact ByteString and falls back to a full
copy for
a slice or a multi-fragment one.
ByteReader.remainingData is input.drop(off), so it is not compact once any
byte
has been consumed, which is always the case by the time Inflate runs.
Inflater
and Deflater have taken ByteBuffer input since Java 11 and CRC32 since Java
8,
and the build targets JDK 17.
Modification:
Pass reader.remainingData.asByteBuffer to Inflater.setInput. getRemaining
reports
the buffer's remaining bytes, so the following reader.skip is unchanged.
Feed the
checksums one buffer per fragment with asByteBuffers, which stays copy free
even
for a multi-fragment ByteString, where the singular asByteBuffer would
compact.
Result:
No array copy per inflate round, and no full size copy per gzipped entity.
Measured over a 1 MB input, 200 CRC32 updates: 221 ms via toArrayUnsafe
versus
154 ms via a ByteBuffer for a sliced input, 102 ms versus 98 ms for a
compact
one, so it is never slower and stops allocating a copy of everything
gzipped.
Tests:
- sbt "http-tests / Test / testOnly
org.apache.pekko.http.scaladsl.coding.*" - 78 passed. CoderSpec already covers
these paths well: 'works for any split in prefix + suffix' decodes at every
possible split point, so remainingData is a slice at every offset, plus chunked
decoding and corrupt input
- New CoderSpec case asserts the encoder produces identical output for a
multi-fragment input and a compact one, pinning the per-fragment checksum. It
is a regression guard, not a failing-before test, since the previous code was
also correct
- sbt http/mimaReportBinaryIssues - clean
- scalafmt --mode diff-ref=upstream/main - clean
References:
None - found while auditing ByteString usage across the code base
---
.../scala/org/apache/pekko/http/scaladsl/coding/CoderSpec.scala | 9 +++++++++
.../apache/pekko/http/scaladsl/coding/DeflateCompressor.scala | 2 +-
.../org/apache/pekko/http/scaladsl/coding/GzipCompressor.scala | 5 +++--
3 files changed, 13 insertions(+), 3 deletions(-)
diff --git
a/http-tests/src/test/scala/org/apache/pekko/http/scaladsl/coding/CoderSpec.scala
b/http-tests/src/test/scala/org/apache/pekko/http/scaladsl/coding/CoderSpec.scala
index 708d39aed..a8677ff6d 100644
---
a/http-tests/src/test/scala/org/apache/pekko/http/scaladsl/coding/CoderSpec.scala
+++
b/http-tests/src/test/scala/org/apache/pekko/http/scaladsl/coding/CoderSpec.scala
@@ -107,6 +107,15 @@ abstract class CoderSpec extends AnyWordSpec with
CodecSpecSupport with Inspecto
uncompressed should readAs(largeText)
}
+ "produce the same output for a multi-fragment input as for a compact one"
in {
+ // the checksum is fed one ByteBuffer per fragment, so it must not
depend on how the input is fragmented
+ val fragmented = largeTextBytes.grouped(512).map(chunk =>
ByteString(chunk.toArray)).reduce(_ ++ _)
+ fragmented.isCompact shouldEqual false
+ fragmented shouldEqual largeTextBytes
+
+ ourEncode(fragmented) shouldEqual ourEncode(largeTextBytes)
+ ourDecode(ourEncode(fragmented)) should readAs(largeText)
+ }
"works for any split in prefix + suffix" in {
val compressed = streamEncode(smallTextBytes)
def tryWithPrefixOfSize(prefixSize: Int): Unit = {
diff --git
a/http/src/main/scala/org/apache/pekko/http/scaladsl/coding/DeflateCompressor.scala
b/http/src/main/scala/org/apache/pekko/http/scaladsl/coding/DeflateCompressor.scala
index bd84ebc3d..563335985 100644
---
a/http/src/main/scala/org/apache/pekko/http/scaladsl/coding/DeflateCompressor.scala
+++
b/http/src/main/scala/org/apache/pekko/http/scaladsl/coding/DeflateCompressor.scala
@@ -172,7 +172,7 @@ private[coding] abstract class
DeflateDecompressorBase(maxBytesPerChunk: Int = D
override def canWorkWithPartialData = true
override def parse(reader: ByteStringParser.ByteReader):
ParseResult[ByteString] = {
- inflater.setInput(reader.remainingData.toArray)
+ inflater.setInput(reader.remainingData.asByteBuffer)
val buffer = new Array[Byte](maxBytesPerChunk)
val read = inflater.inflate(buffer)
diff --git
a/http/src/main/scala/org/apache/pekko/http/scaladsl/coding/GzipCompressor.scala
b/http/src/main/scala/org/apache/pekko/http/scaladsl/coding/GzipCompressor.scala
index 78568d09a..287671ea1 100644
---
a/http/src/main/scala/org/apache/pekko/http/scaladsl/coding/GzipCompressor.scala
+++
b/http/src/main/scala/org/apache/pekko/http/scaladsl/coding/GzipCompressor.scala
@@ -41,7 +41,8 @@ private[coding] class GzipCompressor(compressionLevel: Int)
extends DeflateCompr
header() ++ super.finishWithBuffer(buffer) ++ trailer()
private def updateCrc(input: ByteString): Unit = {
- checkSum.update(input.toArrayUnsafe())
+ // one buffer per fragment, so that a multi-fragment ByteString is not
compacted into a copy first
+ input.asByteBuffers.foreach(buffer => checkSum.update(buffer))
bytesRead += input.length
}
private def header(): ByteString =
@@ -131,7 +132,7 @@ private[coding] class GzipDecompressor(
}
private def crc16(data: ByteString) = {
val crc = new CRC32
- crc.update(data.toArrayUnsafe())
+ data.asByteBuffers.foreach(buffer => crc.update(buffer))
crc.getValue.toInt & 0xFFFF
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
