This is an automated email from the ASF dual-hosted git repository. ggregory pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/commons-io.git
commit c832eb20796b25d8ee2437a8663616b1112cd7bd Author: Gary D. Gregory <[email protected]> AuthorDate: Tue Mar 4 07:35:49 2025 -0500 Add FileChannels.contentEquals(ReadableByteChannel, ReadableByteChannel, int) - Improve performance of IOUtils.contentEquals(InputStream, InputStream) by about 13% - Fix JMH Maven invocation - Pick up JMH version from parent POM --- pom.xml | 27 +++++++--- src/changes/changes.xml | 2 + src/main/java/org/apache/commons/io/IOUtils.java | 43 ++------------- .../apache/commons/io/channels/FileChannels.java | 51 ++++++++++++------ .../IOUtilsContentEqualsInputStreamsBenchmark.java | 63 +++++++++++++++++++--- 5 files changed, 115 insertions(+), 71 deletions(-) diff --git a/pom.xml b/pom.xml index d89ac7962..45f54100f 100644 --- a/pom.xml +++ b/pom.xml @@ -101,13 +101,7 @@ file comparators, endian transformation classes, and much more. <dependency> <groupId>org.openjdk.jmh</groupId> <artifactId>jmh-core</artifactId> - <version>${jmh.version}</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.openjdk.jmh</groupId> - <artifactId>jmh-generator-annprocess</artifactId> - <version>${jmh.version}</version> + <version>${commons.jmh.version}</version> <scope>test</scope> </dependency> </dependencies> @@ -148,7 +142,6 @@ file comparators, endian transformation classes, and much more. </commons.osgi.import> <commons.scmPubUrl>https://svn.apache.org/repos/infra/websites/production/commons/content/proper/commons-io/</commons.scmPubUrl> <commons.scmPubCheckoutDirectory>site-content</commons.scmPubCheckoutDirectory> - <jmh.version>1.37</jmh.version> <commons.bytebuddy.version>1.17.2</commons.bytebuddy.version> <japicmp.skip>false</japicmp.skip> <commons.release.isDistModule>true</commons.release.isDistModule> @@ -357,13 +350,31 @@ file comparators, endian transformation classes, and much more. </properties> </profile> <profile> + <!-- Profile to build and run the benchmarks. Use 'mvn test -Pbenchmark', and add '-Dbenchmark=foo' to run only the foo benchmark --> <id>benchmark</id> + <dependencies> + <dependency> + <groupId>org.openjdk.jmh</groupId> + <artifactId>jmh-generator-annprocess</artifactId> + <version>${commons.jmh.version}</version> + <scope>test</scope> + </dependency> + </dependencies> <properties> <skipTests>true</skipTests> <benchmark>org.apache</benchmark> </properties> <build> <plugins> + <plugin> + <artifactId>maven-compiler-plugin</artifactId> + <version>${commons.compiler.version}</version> + <configuration combine.self="override"> + <testIncludes> + <testInclude>**/*</testInclude> + </testIncludes> + </configuration> + </plugin> <plugin> <groupId>org.codehaus.mojo</groupId> <artifactId>exec-maven-plugin</artifactId> diff --git a/src/changes/changes.xml b/src/changes/changes.xml index c6c96c2e5..a92b183f8 100644 --- a/src/changes/changes.xml +++ b/src/changes/changes.xml @@ -64,6 +64,7 @@ The <action> type attribute can be add,update,fix,remove. <action dev="ggregory" type="fix" due-to="Gary Gregory">Avoid unnecessary boxing and unboxing of int values in UncheckedFilterReader.read().</action> <action dev="ggregory" type="fix" due-to="Gary Gregory">FileChannels.contentEquals(FileChannel, FileChannel, int) can return false when comparing a non-blocking channel.</action> <action dev="ggregory" type="fix" due-to="Gary Gregory">Deprecate FileChannels.contentEquals(FileChannel, FileChannel, int) in favor of FileChannels.contentEquals(SeekableByteChannel, SeekableByteChannel, int).</action> + <action dev="ggregory" type="fix" due-to="Gary Gregory">Improve performance of IOUtils.contentEquals(InputStream, InputStream) by about 13%.</action> <!-- ADD --> <action dev="ggregory" type="add" issue="IO-860" due-to="Nico Strecker, Gary Gregory">Add ThrottledInputStream.Builder.setMaxBytes(long, ChronoUnit).</action> <action dev="ggregory" type="add" due-to="Gary Gregory">Add IOIterable.</action> @@ -80,6 +81,7 @@ The <action> type attribute can be add,update,fix,remove. <action dev="ggregory" type="add" due-to="Gary Gregory">Add IOBooleanSupplier.</action> <action dev="ggregory" type="add" due-to="Gary Gregory">Add Uncheck.getAsBoolean(IOBooleanSupplier).</action> <action dev="ggregory" type="add" due-to="Gary Gregory">Add FileChannels.contentEquals(SeekableByteChannel, SeekableByteChannel, int).</action> + <action dev="ggregory" type="add" due-to="Gary Gregory">Add FileChannels.contentEquals(ReadableByteChannel, ReadableByteChannel, int).</action> <!-- UPDATE --> <action dev="ggregory" type="update" due-to="Dependabot, Gary Gregory">Bump commons.bytebuddy.version from 1.15.10 to 1.17.2 #710, #715, #720.</action> <action dev="ggregory" type="update" due-to="Gary Gregory">Bump commons-codec:commons-codec from 1.17.1 to 1.18.0. #717.</action> diff --git a/src/main/java/org/apache/commons/io/IOUtils.java b/src/main/java/org/apache/commons/io/IOUtils.java index 2717aa01e..76812b431 100644 --- a/src/main/java/org/apache/commons/io/IOUtils.java +++ b/src/main/java/org/apache/commons/io/IOUtils.java @@ -61,6 +61,7 @@ import java.util.stream.Stream; import java.util.zip.InflaterInputStream; +import org.apache.commons.io.channels.FileChannels; import org.apache.commons.io.function.IOConsumer; import org.apache.commons.io.function.IOSupplier; import org.apache.commons.io.function.IOTriFunction; @@ -903,51 +904,17 @@ public static long consume(final Reader input) throws IOException { * exist, false otherwise * @throws IOException if an I/O error occurs */ + @SuppressWarnings("resource") // Caller closes input streams public static boolean contentEquals(final InputStream input1, final InputStream input2) throws IOException { - // Before making any changes, please test with - // org.apache.commons.io.jmh.IOUtilsContentEqualsInputStreamsBenchmark + // Before making any changes, please test with org.apache.commons.io.jmh.IOUtilsContentEqualsInputStreamsBenchmark if (input1 == input2) { return true; } if (input1 == null || input2 == null) { return false; } - - // reuse one - final byte[] array1 = getScratchByteArray(); - // allocate another - final byte[] array2 = byteArray(); - int pos1; - int pos2; - int count1; - int count2; - while (true) { - pos1 = 0; - pos2 = 0; - for (int index = 0; index < DEFAULT_BUFFER_SIZE; index++) { - if (pos1 == index) { - do { - count1 = input1.read(array1, pos1, DEFAULT_BUFFER_SIZE - pos1); - } while (count1 == 0); - if (count1 == EOF) { - return pos2 == index && input2.read() == EOF; - } - pos1 += count1; - } - if (pos2 == index) { - do { - count2 = input2.read(array2, pos2, DEFAULT_BUFFER_SIZE - pos2); - } while (count2 == 0); - if (count2 == EOF) { - return pos1 == index && input1.read() == EOF; - } - pos2 += count2; - } - if (array1[index] != array2[index]) { - return false; - } - } - } + // We do not close FileChannels because that closes the owning InputStream. + return FileChannels.contentEquals(Channels.newChannel(input1), Channels.newChannel(input2), DEFAULT_BUFFER_SIZE); } // TODO Consider making public diff --git a/src/main/java/org/apache/commons/io/channels/FileChannels.java b/src/main/java/org/apache/commons/io/channels/FileChannels.java index 75f34d1c4..4f9eb60b4 100644 --- a/src/main/java/org/apache/commons/io/channels/FileChannels.java +++ b/src/main/java/org/apache/commons/io/channels/FileChannels.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.channels.ReadableByteChannel; import java.nio.channels.SeekableByteChannel; import java.util.Objects; @@ -33,10 +34,10 @@ public final class FileChannels { /** - * Tests if two FileChannel contents are equal starting at their respective current positions. + * Tests if two file channel contents are equal starting at their respective current positions. * - * @param channel1 A FileChannel. - * @param channel2 Another FileChannel. + * @param channel1 A file channel. + * @param channel2 Another file channel. * @param bufferCapacity The two internal buffer capacities, in bytes. * @return true if the contents of both RandomAccessFiles are equal, false otherwise. * @throws IOException if an I/O error occurs. @@ -49,10 +50,6 @@ public static boolean contentEquals(final FileChannel channel1, final FileChanne /** * Tests if two readable byte channel contents are equal starting at their respective current positions. - * <p> - * If a file channel is a non-blocking file channel, it may return 0 bytes read for any given call. In order to avoid waiting forever when trying again, a - * timeout Duration can be specified, which when met, throws an IOException. - * </p> * * @param channel1 A readable byte channel. * @param channel2 Another readable byte channel. @@ -61,20 +58,12 @@ public static boolean contentEquals(final FileChannel channel1, final FileChanne * @throws IOException if an I/O error occurs or the timeout is met. * @since 2.19.0 */ - public static boolean contentEquals(final SeekableByteChannel channel1, final SeekableByteChannel channel2, final int bufferCapacity) throws IOException { + public static boolean contentEquals(final ReadableByteChannel channel1, final ReadableByteChannel channel2, final int bufferCapacity) throws IOException { + // Before making any changes, please test with org.apache.commons.io.jmh.IOUtilsContentEqualsInputStreamsBenchmark // Short-circuit test if (Objects.equals(channel1, channel2)) { return true; } - // Short-circuit test - final long size1 = size(channel1); - final long size2 = size(channel2); - if (size1 != size2) { - return false; - } - if (size1 == 0 && size2 == 0) { - return true; - } // Dig in and do the work final ByteBuffer byteBuffer1 = ByteBuffer.allocateDirect(bufferCapacity); final ByteBuffer byteBuffer2 = ByteBuffer.allocateDirect(bufferCapacity); @@ -82,6 +71,7 @@ public static boolean contentEquals(final SeekableByteChannel channel1, final Se int numRead2 = 0; boolean read0On1 = false; boolean read0On2 = false; + // If a channel is a non-blocking channel, it may return 0 bytes read for any given call. while (true) { if (!read0On2) { numRead1 = channel1.read(byteBuffer1); @@ -110,6 +100,33 @@ public static boolean contentEquals(final SeekableByteChannel channel1, final Se } } + /** + * Tests if two seekable byte channel contents are equal starting at their respective current positions. + * <p> + * If the two channels have different sizes, no content comparison takes place, and this method returns false. + * </p> + * + * @param channel1 A seekable byte channel. + * @param channel2 Another seekable byte channel. + * @param bufferCapacity The two internal buffer capacities, in bytes. + * @return true if the contents of both RandomAccessFiles are equal, false otherwise. + * @throws IOException if an I/O error occurs or the timeout is met. + * @since 2.19.0 + */ + public static boolean contentEquals(final SeekableByteChannel channel1, final SeekableByteChannel channel2, final int bufferCapacity) throws IOException { + // Short-circuit test + if (Objects.equals(channel1, channel2)) { + return true; + } + // Short-circuit test + final long size1 = size(channel1); + final long size2 = size(channel2); + if (size1 != size2) { + return false; + } + return size1 == 0 && size2 == 0 || contentEquals((ReadableByteChannel) channel1, channel2, bufferCapacity); + } + private static long size(final SeekableByteChannel channel) throws IOException { return channel != null ? channel.size() : 0; } diff --git a/src/test/java/org/apache/commons/io/jmh/IOUtilsContentEqualsInputStreamsBenchmark.java b/src/test/java/org/apache/commons/io/jmh/IOUtilsContentEqualsInputStreamsBenchmark.java index 6823e8445..08eac5d43 100644 --- a/src/test/java/org/apache/commons/io/jmh/IOUtilsContentEqualsInputStreamsBenchmark.java +++ b/src/test/java/org/apache/commons/io/jmh/IOUtilsContentEqualsInputStreamsBenchmark.java @@ -24,10 +24,12 @@ import java.io.BufferedInputStream; import java.io.IOException; import java.io.InputStream; +import java.nio.channels.Channels; import java.nio.charset.Charset; import java.util.concurrent.TimeUnit; import org.apache.commons.io.IOUtils; +import org.apache.commons.io.channels.FileChannels; import org.apache.commons.lang3.StringUtils; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; @@ -43,14 +45,19 @@ /** * Test different implementations of {@link IOUtils#contentEquals(InputStream, InputStream)}. * - * <pre> - * IOUtilsContentEqualsInputStreamsBenchmark.testFileCurrent avgt 5 1518342.821 ▒ 201890.705 ns/op - * IOUtilsContentEqualsInputStreamsBenchmark.testFilePr118 avgt 5 1578606.938 ▒ 66980.718 ns/op - * IOUtilsContentEqualsInputStreamsBenchmark.testFileRelease_2_8_0 avgt 5 2439163.068 ▒ 265765.294 ns/op - * IOUtilsContentEqualsInputStreamsBenchmark.testStringCurrent avgt 5 10389834700.000 ▒ 330301175.219 ns/op - * IOUtilsContentEqualsInputStreamsBenchmark.testStringPr118 avgt 5 10890915400.000 ▒ 3251289634.067 ns/op - * IOUtilsContentEqualsInputStreamsBenchmark.testStringRelease_2_8_0 avgt 5 12522802960.000 ▒ 111147669.527 ns/op - * </pre> + * <pre>{@code +Benchmark Mode Cnt Score Error Units +IOUtilsContentEqualsInputStreamsBenchmark.testFileChannels avgt 5 65105.350 ± 2655.812 ns/op +IOUtilsContentEqualsInputStreamsBenchmark.testFileCurrent avgt 5 75452.987 ± 260.088 ns/op +IOUtilsContentEqualsInputStreamsBenchmark.testFilePr118 avgt 5 74346.141 ± 2138.149 ns/op +IOUtilsContentEqualsInputStreamsBenchmark.testFileRelease_2_8_0 avgt 5 157246.303 ± 215.369 ns/op +IOUtilsContentEqualsInputStreamsBenchmark.testStringCurrent avgt 5 623344988.622 ± 574407721.150 ns/op +IOUtilsContentEqualsInputStreamsBenchmark.testStringFileChannels avgt 5 132847058.786 ± 1760007.730 ns/op +IOUtilsContentEqualsInputStreamsBenchmark.testStringPr118 avgt 5 459079096.521 ± 512827244.936 ns/op +IOUtilsContentEqualsInputStreamsBenchmark.testStringRelease_2_8_0 avgt 5 2555773583.300 ± 18112219.764 ns/op + +[INFO] Finished at: 2025-03-03T21:11:56-05:00 + * }</pre> */ @BenchmarkMode(Mode.AverageTime) @OutputTimeUnit(TimeUnit.NANOSECONDS) @@ -101,6 +108,16 @@ public static boolean contentEquals_release_2_8_0(final InputStream input1, fina } + public static boolean contentEqualsFileChannels(final InputStream input1, final InputStream input2) throws IOException { + if (input1 == input2) { + return true; + } + if (input1 == null || input2 == null) { + return false; + } + return FileChannels.contentEquals(Channels.newChannel(input1), Channels.newChannel(input2), IOUtils.DEFAULT_BUFFER_SIZE); + } + public static boolean contentEqualsPr118(final InputStream input1, final InputStream input2) throws IOException { if (input1 == input2) { return true; @@ -144,6 +161,24 @@ public static boolean contentEqualsPr118(final InputStream input1, final InputSt } } + @Benchmark + public boolean[] testFileChannels() throws IOException { + final boolean[] res = new boolean[3]; + try (InputStream input1 = getClass().getResourceAsStream(TEST_PATH_A); + InputStream input2 = getClass().getResourceAsStream(TEST_PATH_B)) { + res[0] = contentEqualsFileChannels(input1, input1); + } + try (InputStream input1 = getClass().getResourceAsStream(TEST_PATH_A); + InputStream input2 = getClass().getResourceAsStream(TEST_PATH_A)) { + res[1] = contentEqualsFileChannels(input1, input2); + } + try (InputStream input1 = getClass().getResourceAsStream(TEST_PATH_16K_A); + InputStream input2 = getClass().getResourceAsStream(TEST_PATH_16K_A_COPY)) { + res[2] = contentEqualsFileChannels(input1, input2); + } + return res; + } + @Benchmark public boolean[] testFileCurrent() throws IOException { final boolean[] res = new boolean[3]; @@ -210,6 +245,18 @@ public void testStringCurrent(final Blackhole blackhole) throws IOException { } } + @Benchmark + public void testStringFileChannels(final Blackhole blackhole) throws IOException { + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 5; j++) { + try (InputStream input1 = IOUtils.toInputStream(STRINGS[i], DEFAULT_CHARSET); + InputStream input2 = IOUtils.toInputStream(STRINGS[j], DEFAULT_CHARSET)) { + blackhole.consume(contentEqualsFileChannels(input1, input2)); + } + } + } + } + @Benchmark public void testStringPr118(final Blackhole blackhole) throws IOException { for (int i = 0; i < 5; i++) {
