This is an automated email from the ASF dual-hosted git repository. chibenwa pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/james-mime4j.git
commit c9d4e103ff1feecfdd2df8546ef96d7531d04c12 Author: Benoit TELLIER <[email protected]> AuthorDate: Thu Nov 19 06:26:26 2026 +0100 [PERF] Base64OutputStream: encode whole lines with the JDK array encoder The wrapped JDK stream encoder encodes in Java and writes each base64 char with out.write(int): 70% of writeMessage. Blocks of whole lines (about 3 KB) are now encoded with Base64.Encoder#encode(byte[], byte[]), which is intrinsic, and written in one call. A partial line or a carry goes through a group encoder writing to a buffer, flushed once per write call. The output is byte for byte the same: line length rounded down to a multiple of 4, separator between lines, a trailing separator only when something was written. A test compares it to the JDK stream encoder for random sizes, write splits, line lengths and separators. --- .../james/mime4j/codec/Base64OutputStream.java | 287 ++++++++++++++++++--- .../james/mime4j/codec/Base64OutputStreamTest.java | 85 ++++++ 2 files changed, 332 insertions(+), 40 deletions(-) diff --git a/core/src/main/java/org/apache/james/mime4j/codec/Base64OutputStream.java b/core/src/main/java/org/apache/james/mime4j/codec/Base64OutputStream.java index f213a81f..759daeb5 100644 --- a/core/src/main/java/org/apache/james/mime4j/codec/Base64OutputStream.java +++ b/core/src/main/java/org/apache/james/mime4j/codec/Base64OutputStream.java @@ -50,7 +50,56 @@ public class Base64OutputStream extends OutputStream { 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/' }; - private final OutputStream delegate; + // Input bytes encoded per call of the JDK encoder: a whole number of lines, about 3 KB. + private static final int BLOCK_INPUT_TARGET = 3072; + + // Room for the output of the per-group path before it is written out, besides one line separator. + private static final int OUT_BUFFER_SIZE = 1024; + + // Line separators up to this length use the buffers of the thread + private static final int SHARED_MAX_SEPARATOR = 16; + + /** + * The work buffers. They hold nothing between two calls of the stream, so a thread shares one set + * between its streams; a stream writing into another one on the same thread finds it in use and + * takes its own. + */ + private static final class Buffers { + final byte[] out; + byte[] block; + byte[] encodedBlock; + boolean inUse; + + Buffers(int outSize) { + this.out = new byte[outSize]; + } + } + + private static final ThreadLocal<Buffers> THREAD_BUFFERS = + ThreadLocal.withInitial(() -> new Buffers(OUT_BUFFER_SIZE + SHARED_MAX_SEPARATOR)); + + private final OutputStream out; + private final byte[] lineSeparator; + // Encoded chars per line, a multiple of 4; 0 or less: no line breaks + private final int lineMax; + // Input bytes encoded at once by the JDK encoder, from a line start + private final int blockInput; + private final Base64.Encoder blockEncoder; + + private final int encodedBlockLength; + + private final byte[] carry = new byte[3]; + private int leftover = 0; + // Chars written on the current line. A separator is written before the next group once the line is full. + private int linePos = 0; + // The buffers of the call in progress, null between calls + private Buffers buffers; + private byte[] outBuffer; + private int outPos = 0; + // Used instead of the buffers of the thread when they are in use, or do not fit + private Buffers ownBuffers; + private boolean wroteAny = false; + private boolean closed = false; /** * Creates a <code>Base64OutputStream</code> that writes the encoded data @@ -69,7 +118,7 @@ public class Base64OutputStream extends OutputStream { * to the given output stream using the given line length and the default * line separator (CRLF). * <p> - * The given line length will be rounded up to the nearest multiple of 4. If + * The given line length will be rounded down to the nearest multiple of 4. If * the line length is zero then the output will not be split into lines. * * @param out @@ -86,7 +135,7 @@ public class Base64OutputStream extends OutputStream { * to the given output stream using the given line length and line * separator. * <p> - * The given line length will be rounded up to the nearest multiple of 4. If + * The given line length will be rounded down to the nearest multiple of 4. If * the line length is zero then the output will not be split into lines and * the line separator is ignored. * <p> @@ -101,74 +150,232 @@ public class Base64OutputStream extends OutputStream { * line separator to use. */ public Base64OutputStream(OutputStream out, int lineLength, byte[] lineSeparator) { - ExtraCrlfOutputStream wrapped = new ExtraCrlfOutputStream(out, lineSeparator); - this.delegate = Base64.getMimeEncoder(lineLength, lineSeparator).wrap(wrapped); + // Validates the separator as before, and gives the output of whole lines + Base64.Encoder mimeEncoder = Base64.getMimeEncoder(lineLength, lineSeparator); + this.out = out; + this.lineSeparator = lineSeparator.clone(); + // As the JDK MIME encoder: rounded down to a multiple of 4 + this.lineMax = lineLength & ~0b11; + if (lineMax > 0) { + int lineInput = lineMax / 4 * 3; + this.blockInput = Math.max(1, BLOCK_INPUT_TARGET / lineInput) * lineInput; + this.blockEncoder = mimeEncoder; + int lines = blockInput / lineInput; + this.encodedBlockLength = lines * lineMax + (lines - 1) * this.lineSeparator.length; + } else { + this.blockInput = BLOCK_INPUT_TARGET; + this.blockEncoder = Base64.getEncoder(); + this.encodedBlockLength = blockInput / 3 * 4; + } + } + + private void acquire() { + Buffers shared = THREAD_BUFFERS.get(); + if (!shared.inUse && lineSeparator.length <= SHARED_MAX_SEPARATOR + && blockInput <= BLOCK_INPUT_TARGET) { + shared.inUse = true; + buffers = shared; + } else { + if (ownBuffers == null) { + ownBuffers = new Buffers(OUT_BUFFER_SIZE + lineSeparator.length); + } + buffers = ownBuffers; + } + outBuffer = buffers.out; + } + + private void release() { + if (buffers != null) { + buffers.inUse = false; + buffers = null; + outBuffer = null; + // nothing is left there, unless the underlying stream failed + outPos = 0; + } } @Override public void write(int i) throws IOException { - delegate.write(i); + ensureOpen(); + carry[leftover++] = (byte) i; + if (leftover == 3) { + acquire(); + try { + encodeGroup(carry, 0); + leftover = 0; + flushOutBuffer(); + } finally { + release(); + } + } } @Override public void write(byte[] b) throws IOException { - delegate.write(b); + write(b, 0, b.length); } @Override public void write(byte[] b, int off, int len) throws IOException { - delegate.write(b, off, len); + ensureOpen(); + if (off < 0 || len < 0 || len > b.length - off) { + throw new ArrayIndexOutOfBoundsException(); + } + if (len == 0) { + return; + } + if (leftover + len < 3) { + while (len > 0) { + carry[leftover++] = b[off++]; + len--; + } + return; + } + acquire(); + try { + encode(b, off, len); + } finally { + release(); + } + } + + private void encode(byte[] b, int off, int len) throws IOException { + if (leftover > 0) { + while (leftover < 3 && len > 0) { + carry[leftover++] = b[off++]; + len--; + } + if (leftover < 3) { + return; + } + encodeGroup(carry, 0); + leftover = 0; + } + while (len >= 3) { + if (len >= blockInput && atLineStart()) { + flushOutBuffer(); + encodeBlock(b, off); + off += blockInput; + len -= blockInput; + } else { + encodeGroup(b, off); + off += 3; + len -= 3; + } + } + while (len > 0) { + carry[leftover++] = b[off++]; + len--; + } + flushOutBuffer(); } @Override public void flush() throws IOException { - delegate.flush(); + // The output buffer is written at the end of each call + out.flush(); } @Override public void close() throws IOException { - delegate.close(); + if (closed) { + return; + } + closed = true; + if (leftover > 0) { + acquire(); + try { + writeLastGroup(); + } finally { + release(); + } + } + // As the last line has no separator, one is added once something was written + if (wroteAny) { + out.write(lineSeparator); + } } - private static class ExtraCrlfOutputStream extends OutputStream { - private final OutputStream delegate; - private final byte[] lineSeparator; - private boolean appendExtraCrlf; + /** The one or two bytes left, padded. */ + private void writeLastGroup() throws IOException { + int b0 = carry[0] & 0xff; + int b1 = leftover == 2 ? carry[1] & 0xff : 0; + beforeGroup(); + outBuffer[outPos++] = BASE64_TABLE[b0 >> 2]; + outBuffer[outPos++] = BASE64_TABLE[(b0 << 4 | b1 >> 4) & 0x3f]; + outBuffer[outPos++] = leftover == 2 ? BASE64_TABLE[(b1 << 2) & 0x3f] : (byte) '='; + outBuffer[outPos++] = '='; + leftover = 0; + flushOutBuffer(); + } - private ExtraCrlfOutputStream(OutputStream delegate, byte[] lineSeparator) { - this.delegate = delegate; - this.lineSeparator = lineSeparator; - this.appendExtraCrlf = false; + private void ensureOpen() throws IOException { + if (closed) { + throw new IOException("Stream is closed"); } + } - @Override - public void write(int i) throws IOException { - delegate.write(i); - appendExtraCrlf = true; - } + private boolean atLineStart() { + return lineMax <= 0 || linePos == 0 || linePos == lineMax; + } - @Override - public void write(byte[] b) throws IOException { - delegate.write(b); - appendExtraCrlf |= b.length > 0; + /** + * Makes room for a group of 4 chars in the output buffer, after the pending line separator if the line is full. + */ + private void beforeGroup() throws IOException { + if (outPos > outBuffer.length - 4 - lineSeparator.length) { + flushOutBuffer(); } - - @Override - public void write(byte[] b, int off, int len) throws IOException { - delegate.write(b, off, len); - appendExtraCrlf |= len> 0; + if (lineMax > 0) { + if (linePos == lineMax) { + System.arraycopy(lineSeparator, 0, outBuffer, outPos, lineSeparator.length); + outPos += lineSeparator.length; + linePos = 0; + } + linePos += 4; } + } + + private void encodeGroup(byte[] b, int off) throws IOException { + beforeGroup(); + int bits = (b[off] & 0xff) << 16 | (b[off + 1] & 0xff) << 8 | (b[off + 2] & 0xff); + outBuffer[outPos++] = BASE64_TABLE[bits >>> 18]; + outBuffer[outPos++] = BASE64_TABLE[(bits >>> 12) & 0x3f]; + outBuffer[outPos++] = BASE64_TABLE[(bits >>> 6) & 0x3f]; + outBuffer[outPos++] = BASE64_TABLE[bits & 0x3f]; + } - @Override - public void flush() throws IOException { - delegate.flush(); + /** + * Encodes <code>blockInput</code> bytes from a line start with the (intrinsic) array encoder of the JDK: + * whole lines, separated, without a trailing separator. + */ + private void encodeBlock(byte[] b, int off) throws IOException { + byte[] block = buffers.block; + byte[] encodedBlock = buffers.encodedBlock; + if (block == null || block.length != blockInput || encodedBlock.length != encodedBlockLength) { + // The JDK encodes the whole array: it has the exact block length + block = new byte[blockInput]; + encodedBlock = new byte[encodedBlockLength]; + buffers.block = block; + buffers.encodedBlock = encodedBlock; + } + if (lineMax > 0 && linePos == lineMax) { + out.write(lineSeparator); } + System.arraycopy(b, off, block, 0, blockInput); + int n = blockEncoder.encode(block, encodedBlock); + out.write(encodedBlock, 0, n); + wroteAny = true; + if (lineMax > 0) { + linePos = lineMax; + } + } - @Override - public void close() throws IOException { - if (appendExtraCrlf) { - delegate.write(lineSeparator); - } + private void flushOutBuffer() throws IOException { + if (outPos > 0) { + out.write(outBuffer, 0, outPos); + outPos = 0; + wroteAny = true; } } } diff --git a/core/src/test/java/org/apache/james/mime4j/codec/Base64OutputStreamTest.java b/core/src/test/java/org/apache/james/mime4j/codec/Base64OutputStreamTest.java index 0600781c..2793b636 100644 --- a/core/src/test/java/org/apache/james/mime4j/codec/Base64OutputStreamTest.java +++ b/core/src/test/java/org/apache/james/mime4j/codec/Base64OutputStreamTest.java @@ -27,6 +27,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; import java.io.UnsupportedEncodingException; +import java.util.Base64; +import java.util.Random; public class Base64OutputStreamTest { @@ -223,4 +225,87 @@ public class Base64OutputStreamTest { return null; } } + @Test + public void encodingInChunksShouldWriteWhatTheJdkStreamEncoderWrites() throws IOException { + Random random = new Random(42); + int[] lineLengths = {76, 0, -1, 3, 4, 5, 64, 77, 1000, 5000}; + byte[][] separators = {{'\r', '\n'}, {'\n'}, {}, {'\r', '\n', ' ', '\t'}}; + int[] sizes = {0, 1, 2, 3, 56, 57, 58, 3647, 3648, 3649, 3072, 4000, 10000, 70001}; + for (int lineLength : lineLengths) { + for (byte[] separator : separators) { + for (int size : sizes) { + byte[] content = new byte[size]; + random.nextBytes(content); + for (int attempt = 0; attempt < 4; attempt++) { + assertSameAsReference(content, lineLength, separator, random, attempt); + } + } + } + } + } + + @Test + public void aStreamWritingIntoAnotherOnTheSameThreadShouldEncodeBothTimes() throws IOException { + byte[] content = new byte[100000]; + new Random(43).nextBytes(content); + + ByteArrayOutputStream once = new ByteArrayOutputStream(); + try (Base64OutputStream encoder = new Base64OutputStream(once)) { + encoder.write(content); + } + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + try (Base64OutputStream encoder = new Base64OutputStream(expected)) { + encoder.write(once.toByteArray()); + } + + ByteArrayOutputStream actual = new ByteArrayOutputStream(); + try (Base64OutputStream outer = new Base64OutputStream(actual); + Base64OutputStream inner = new Base64OutputStream(outer)) { + for (int pos = 0; pos < content.length; pos += 5000) { + inner.write(content, pos, Math.min(5000, content.length - pos)); + } + } + + Assert.assertArrayEquals(expected.toByteArray(), actual.toByteArray()); + } + + private void assertSameAsReference(byte[] content, int lineLength, byte[] separator, Random random, int attempt) throws IOException { + ByteArrayOutputStream expected = new ByteArrayOutputStream(); + // The previous implementation: the JDK stream encoder, and a trailing separator if anything was written + ByteArrayOutputStream encoded = new ByteArrayOutputStream(); + OutputStream reference = Base64.getMimeEncoder(lineLength, separator).wrap(encoded); + reference.write(content); + reference.close(); + expected.write(encoded.toByteArray()); + if (encoded.size() > 0) { + expected.write(separator); + } + + ByteArrayOutputStream actual = new ByteArrayOutputStream(); + Base64OutputStream encoder = new Base64OutputStream(actual, lineLength, separator); + int pos = 0; + while (pos < content.length) { + int split; + if (attempt == 0) { + split = content.length - pos; + } else if (attempt == 1) { + split = 1 + random.nextInt(Math.min(10, content.length - pos)); + } else { + split = 1 + random.nextInt(content.length - pos); + } + if (split == 1 && random.nextBoolean()) { + encoder.write(content[pos]); + } else { + encoder.write(content, pos, split); + } + if (random.nextInt(8) == 0) { + encoder.flush(); + } + pos += split; + } + encoder.close(); + + Assert.assertArrayEquals("lineLength " + lineLength + " separator " + separator.length + " size " + content.length + " attempt " + attempt, + expected.toByteArray(), actual.toByteArray()); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
