LOG4J2-1274 add TextEncoderHelper with initial unit tests
Project: http://git-wip-us.apache.org/repos/asf/logging-log4j2/repo Commit: http://git-wip-us.apache.org/repos/asf/logging-log4j2/commit/27619e08 Tree: http://git-wip-us.apache.org/repos/asf/logging-log4j2/tree/27619e08 Diff: http://git-wip-us.apache.org/repos/asf/logging-log4j2/diff/27619e08 Branch: refs/heads/LOG4J2-1278-gc-free-logger Commit: 27619e0812ceaf4ead6b086268505ee55459e7eb Parents: 9594f5b Author: rpopma <[email protected]> Authored: Sun Feb 21 16:49:31 2016 +0900 Committer: rpopma <[email protected]> Committed: Sun Feb 21 16:49:31 2016 +0900 ---------------------------------------------------------------------- .../log4j/core/layout/TextEncoderHelper.java | 136 ++++++++++++ .../core/layout/SpyByteBufferDestination.java | 63 ++++++ .../core/layout/TextEncoderHelperTest.java | 216 +++++++++++++++++++ .../apache/logging/log4j/perf/nogc/Encoder.java | 24 --- 4 files changed, 415 insertions(+), 24 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/27619e08/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java ---------------------------------------------------------------------- diff --git a/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java new file mode 100644 index 0000000..79f0123 --- /dev/null +++ b/log4j-core/src/main/java/org/apache/logging/log4j/core/layout/TextEncoderHelper.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ +package org.apache.logging.log4j.core.layout; + +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.CharacterCodingException; +import java.nio.charset.Charset; +import java.nio.charset.CharsetEncoder; +import java.nio.charset.CoderResult; +import java.nio.charset.CodingErrorAction; +import java.util.Objects; + +/** + * Helper class to encode text to binary data without allocating temporary objects. + * + * @since 2.6 + */ +class TextEncoderHelper { + private static final int DEFAULT_BUFFER_SIZE = 2048; + + private final Charset charset; + private final CharBuffer cachedCharBuffer; + private final CharsetEncoder charsetEncoder; + + public TextEncoderHelper(final Charset charset) { + this(charset, DEFAULT_BUFFER_SIZE); + } + + public TextEncoderHelper(final Charset charset, final int bufferSize) { + this.charset = Objects.requireNonNull(charset, "charset"); + this.charsetEncoder = charset.newEncoder().onMalformedInput(CodingErrorAction.REPLACE) + .onUnmappableCharacter(CodingErrorAction.REPLACE); + this.cachedCharBuffer = CharBuffer.wrap(new char[bufferSize]); + } + + public void encodeText(final StringBuilder text, final ByteBufferDestination destination) { + charsetEncoder.reset(); + ByteBuffer byteBuf = destination.getByteBuffer(); + final CharBuffer charBuf = getCachedCharBuffer(); + int start = 0; + int todoChars = text.length(); + boolean endOfInput = true; + do { + charBuf.clear(); // reset character buffer position to zero, limit to capacity + int copied = copy(text, start, charBuf); + start += copied; + todoChars -= copied; + endOfInput = todoChars <= 0; + + charBuf.flip(); // prepare for reading: set limit to position, position to zero + byteBuf = encode(charBuf, endOfInput, destination, byteBuf); + } while (!endOfInput); + } + + private ByteBuffer encode(final CharBuffer charBuf, final boolean endOfInput, + final ByteBufferDestination destination, ByteBuffer byteBuf) { + try { + byteBuf = encodeAsMuchAsPossible(charBuf, endOfInput, destination, byteBuf); + if (endOfInput) { + byteBuf = flushRemainingBytes(destination, byteBuf); + } + } catch (final CharacterCodingException ex) { + throw new IllegalStateException(ex); + } + return byteBuf; + } + + private ByteBuffer encodeAsMuchAsPossible(final CharBuffer charBuf, final boolean endOfInput, + final ByteBufferDestination destination, ByteBuffer byteBuf) throws CharacterCodingException { + CoderResult result; + do { + result = charsetEncoder.encode(charBuf, byteBuf, endOfInput); + if (result.isOverflow()) { // byteBuf full + // destination consumes contents + // and returns byte buffer with more capacity + byteBuf = destination.drain(byteBuf); + } + } while (result.isOverflow()); // byteBuf has been drained: retry + if (!result.isUnderflow()) { // we should have fully read the char buffer contents + result.throwException(); + } + return byteBuf; + } + + private ByteBuffer flushRemainingBytes(final ByteBufferDestination destination, ByteBuffer byteBuf) + throws CharacterCodingException { + CoderResult result; + do { + // write any final bytes to the output buffer once the overall input sequence has been read + result = charsetEncoder.flush(byteBuf); + if (result.isOverflow()) { // byteBuf full + // destination consumes contents + // and returns byte buffer with more capacity + byteBuf = destination.drain(byteBuf); + } + } while (result.isOverflow()); // byteBuf has been drained: retry + if (!result.isUnderflow()) { // we should have fully flushed the remaining bytes + result.throwException(); + } + return byteBuf; + } + + /** + * Copies characters from the StringBuilder into the CharBuffer, + * starting at the specified offset and ending when either all + * characters have been copied or when the CharBuffer is full. + * + * @return the number of characters that were copied + */ + static int copy(final StringBuilder source, final int offset, final CharBuffer destination) { + final int length = Math.min(source.length() - offset, destination.remaining()); + for (int i = offset; i < offset + length; i++) { + destination.put(source.charAt(i)); + } + return length; + } + + CharBuffer getCachedCharBuffer() { + return cachedCharBuffer; + } +} http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/27619e08/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/SpyByteBufferDestination.java ---------------------------------------------------------------------- diff --git a/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/SpyByteBufferDestination.java b/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/SpyByteBufferDestination.java new file mode 100644 index 0000000..5d8f3b5 --- /dev/null +++ b/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/SpyByteBufferDestination.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ +package org.apache.logging.log4j.core.layout; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.List; + +/** + * ByteBufferDestination for test purposes that counts how much data was added. + */ +public class SpyByteBufferDestination implements ByteBufferDestination { + public final ByteBuffer buffer; + public final ByteBuffer drained; + public final List<Data> drainPoints = new ArrayList<>(); + + public static class Data { + public final int position; + public final int limit; + + public Data(final int position, final int limit) { + this.position = position; + this.limit = limit; + } + + public int length() { + return limit - position; + } + } + + public SpyByteBufferDestination(int bufferSize, int destinationSize) { + buffer = ByteBuffer.wrap(new byte[bufferSize]); + drained = ByteBuffer.wrap(new byte[destinationSize]); + } + + @Override + public ByteBuffer getByteBuffer() { + return buffer; + } + + @Override + public ByteBuffer drain(final ByteBuffer buf) { + buf.flip(); + drainPoints.add(new Data(buf.position(), buf.limit())); + drained.put(buf); + buf.clear(); + return buf; + } +} http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/27619e08/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/TextEncoderHelperTest.java ---------------------------------------------------------------------- diff --git a/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/TextEncoderHelperTest.java b/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/TextEncoderHelperTest.java new file mode 100644 index 0000000..5ba6a33 --- /dev/null +++ b/log4j-core/src/test/java/org/apache/logging/log4j/core/layout/TextEncoderHelperTest.java @@ -0,0 +1,216 @@ +package org.apache.logging.log4j.core.layout;/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ + +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; + +import org.junit.Test; + +import static org.junit.Assert.*; + +/** + * Tests the {@code TextEncoderHelper} class. + */ +public class TextEncoderHelperTest { + + @Test + public void testEncodeText_TextFitCharBuff_BytesFitByteBuff() throws Exception { + final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 16); + final StringBuilder text = createText(15); + final SpyByteBufferDestination destination = new SpyByteBufferDestination(17, 17); + helper.encodeText(text, destination); + + assertEquals("drained", 0, destination.drainPoints.size()); + assertEquals("destination.buf.pos", text.length(), destination.buffer.position()); + + for (int i = 0; i < text.length(); i++) { + assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(i)); + } + } + + @Test + public void testEncodeText_TextFitCharBuff_BytesDontFitByteBuff() throws Exception { + final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 16); + final StringBuilder text = createText(15); + final SpyByteBufferDestination destination = new SpyByteBufferDestination(14, 15); + helper.encodeText(text, destination); + + assertEquals("drained", 1, destination.drainPoints.size()); + assertEquals("drained[0].from", 0, destination.drainPoints.get(0).position); + assertEquals("drained[0].to", destination.buffer.capacity(), destination.drainPoints.get(0).limit); + assertEquals("drained[0].length", destination.buffer.capacity(), destination.drainPoints.get(0).length()); + assertEquals("destination.buf.pos", text.length() - destination.buffer.capacity(), + destination.buffer.position()); + + for (int i = 0; i < destination.buffer.capacity(); i++) { + assertEquals("char at " + i, (byte) text.charAt(i), destination.drained.get(i)); + } + for (int i = destination.buffer.capacity(); i < text.length(); i++) { + int bufIx = i - destination.buffer.capacity(); + assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(bufIx)); + } + } + + @Test + public void testEncodeText_TextFitCharBuff_BytesDontFitByteBuff_MultiplePasses() throws Exception { + final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 16); + final StringBuilder text = createText(15); + final SpyByteBufferDestination destination = new SpyByteBufferDestination(4, 20); + helper.encodeText(text, destination); + + assertEquals("drained", 3, destination.drainPoints.size()); + assertEquals("drained[0].from", 0, destination.drainPoints.get(0).position); + assertEquals("drained[0].to", destination.buffer.capacity(), destination.drainPoints.get(0).limit); + assertEquals("drained[0].length", destination.buffer.capacity(), destination.drainPoints.get(0).length()); + assertEquals("drained[1].from", 0, destination.drainPoints.get(1).position); + assertEquals("drained[1].to", destination.buffer.capacity(), destination.drainPoints.get(1).limit); + assertEquals("drained[1].length", destination.buffer.capacity(), destination.drainPoints.get(1).length()); + assertEquals("drained[2].from", 0, destination.drainPoints.get(2).position); + assertEquals("drained[2].to", destination.buffer.capacity(), destination.drainPoints.get(2).limit); + assertEquals("drained[2].length", destination.buffer.capacity(), destination.drainPoints.get(2).length()); + assertEquals("destination.buf.pos", text.length() - 3 * destination.buffer.capacity(), + destination.buffer.position()); + + for (int i = 0; i < 3 * destination.buffer.capacity(); i++) { + assertEquals("char at " + i, (byte) text.charAt(i), destination.drained.get(i)); + } + for (int i = 3 * destination.buffer.capacity(); i < text.length(); i++) { + int bufIx = i - 3 * destination.buffer.capacity(); + assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(bufIx)); + } + } + + @Test + public void testEncodeText_TextDoesntFitCharBuff_BytesFitByteBuff() throws Exception { + final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 4); + final StringBuilder text = createText(15); + final SpyByteBufferDestination destination = new SpyByteBufferDestination(17, 17); + helper.encodeText(text, destination); + + assertEquals("drained", 0, destination.drainPoints.size()); + assertEquals("destination.buf.pos", text.length(), destination.buffer.position()); + + for (int i = 0; i < text.length(); i++) { + assertEquals("char at " + i, (byte) text.charAt(i), destination.buffer.get(i)); + } + } + + @Test + public void testEncodeText_JapaneseTextUtf8DoesntFitCharBuff_BytesFitByteBuff() throws Exception { + final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 4); + final StringBuilder text = new StringBuilder( // æ¥æ¬èªãã¹ãæç« + "\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8\u6587\u7ae0"); + final SpyByteBufferDestination destination = new SpyByteBufferDestination(50, 50); + helper.encodeText(text, destination); + + assertEquals("drained", 0, destination.drainPoints.size()); + destination.drain(destination.getByteBuffer()); + + final byte[] utf8 = text.toString().getBytes(StandardCharsets.UTF_8); + for (int i = 0; i < utf8.length; i++) { + assertEquals("byte at " + i, utf8[i], destination.drained.get(i)); + } + } + + @Test + public void testEncodeText_JapaneseTextShiftJisDoesntFitCharBuff_BytesFitByteBuff() throws Exception { + final Charset SHIFT_JIS = Charset.forName("Shift_JIS"); + final TextEncoderHelper helper = new TextEncoderHelper(SHIFT_JIS, 4); + final StringBuilder text = new StringBuilder( // æ¥æ¬èªãã¹ãæç« + "\u65e5\u672c\u8a9e\u30c6\u30b9\u30c8\u6587\u7ae0"); + final SpyByteBufferDestination destination = new SpyByteBufferDestination(50, 50); + helper.encodeText(text, destination); + + assertEquals("drained", 0, destination.drainPoints.size()); + destination.drain(destination.getByteBuffer()); + + final byte[] bytes = text.toString().getBytes(SHIFT_JIS); + for (int i = 0; i < bytes.length; i++) { + assertEquals("byte at " + i, bytes[i], destination.drained.get(i)); + } + } + + @Test + public void testEncodeText_TextDoesntFitCharBuff_BytesDontFitByteBuff() throws Exception { + final TextEncoderHelper helper = new TextEncoderHelper(StandardCharsets.UTF_8, 16); + // TODO + } + + @Test + public void testCopyCopiesAllDataIfSuffientRemainingSpace() throws Exception { + final CharBuffer buff = CharBuffer.wrap(new char[16]); + final StringBuilder text = createText(15); + final int length = TextEncoderHelper.copy(text, 0, buff); + assertEquals("everything fits", text.length(), length); + for (int i = 0; i < length; i++) { + assertEquals("char at " + i, text.charAt(i), buff.get(i)); + } + assertEquals("position moved by length", text.length(), buff.position()); + } + + @Test + public void testCopyUpToRemainingSpace() throws Exception { + final CharBuffer buff = CharBuffer.wrap(new char[3]); + final StringBuilder text = createText(15); + final int length = TextEncoderHelper.copy(text, 0, buff); + assertEquals("partial copy", buff.capacity(), length); + for (int i = 0; i < length; i++) { + assertEquals("char at " + i, text.charAt(i), buff.get(i)); + } + assertEquals("no space remaining", 0, buff.remaining()); + assertEquals("position at end", buff.capacity(), buff.position()); + } + + @Test + public void testCopyDoesNotWriteBeyondStringText() throws Exception { + final CharBuffer buff = CharBuffer.wrap(new char[5]); + assertEquals("initial buffer position", 0, buff.position()); + final StringBuilder text = createText(2); + final int length = TextEncoderHelper.copy(text, 0, buff); + assertEquals("full copy", text.length(), length); + for (int i = 0; i < length; i++) { + assertEquals("char at " + i, text.charAt(i), buff.get(i)); + } + assertEquals("resulting buffer position", text.length(), buff.position()); + for (int i = length; i < buff.capacity(); i++) { + assertEquals("unset char at " + i, 0, buff.get(i)); + } + } + + @Test + public void testCopyStartsAtBufferPosition() throws Exception { + final CharBuffer buff = CharBuffer.wrap(new char[10]); + final int START_POSITION = 5; + buff.position(START_POSITION); // set start position + final StringBuilder text = createText(15); + final int length = TextEncoderHelper.copy(text, 0, buff); + assertEquals("partial copy", buff.capacity() - START_POSITION, length); + for (int i = 0; i < length; i++) { + assertEquals("char at " + i, text.charAt(i), buff.get(START_POSITION + i)); + } + assertEquals("buffer position at end", buff.capacity(), buff.position()); + } + + private StringBuilder createText(final int length) { + final StringBuilder result = new StringBuilder(length); + for (int i = 0; i < length; i++) { + result.append((char) (' ' + i)); // space=0x20 + } + return result; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/logging-log4j2/blob/27619e08/log4j-perf/src/main/java/org/apache/logging/log4j/perf/nogc/Encoder.java ---------------------------------------------------------------------- diff --git a/log4j-perf/src/main/java/org/apache/logging/log4j/perf/nogc/Encoder.java b/log4j-perf/src/main/java/org/apache/logging/log4j/perf/nogc/Encoder.java deleted file mode 100644 index 9a4da17..0000000 --- a/log4j-perf/src/main/java/org/apache/logging/log4j/perf/nogc/Encoder.java +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache license, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the license for the specific language governing permissions and - * limitations under the license. - */ -package org.apache.logging.log4j.perf.nogc; - -/** - * Encoder interface proposed in LOG4J2-1274.. - */ -public interface Encoder<T> { - void encode(T source, ByteBufferDestination destination); -}
