This is an automated email from the ASF dual-hosted git repository.
ggregory pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/commons-io.git
The following commit(s) were added to refs/heads/master by this push:
new 7002bf28 [IO-764] IOUtils.write() throws
OutOfMemoryError/NegativeArraySizeException while writing big strings
7002bf28 is described below
commit 7002bf2892c2f3c4108cac50671e2e98725adc12
Author: Gary Gregory <[email protected]>
AuthorDate: Tue Apr 5 15:06:23 2022 -0400
[IO-764] IOUtils.write() throws
OutOfMemoryError/NegativeArraySizeException while writing big strings
- Adapt PR #343 by DaGeRe.
- Use a ternary expression.
---
src/changes/changes.xml | 3 +
src/main/java/org/apache/commons/io/IOUtils.java | 12 ++--
.../java/org/apache/commons/io/IOUtilsTest.java | 78 +++++++++++++---------
3 files changed, 55 insertions(+), 38 deletions(-)
diff --git a/src/changes/changes.xml b/src/changes/changes.xml
index 6148792b..c16f03dd 100644
--- a/src/changes/changes.xml
+++ b/src/changes/changes.xml
@@ -165,6 +165,9 @@ The <action> type attribute can be add,update,fix,remove.
<action dev="ggregory" type="fix" due-to="richarda23">
FileFilterTest minor fixes #340.
</action>
+ <action issue="IO-764" dev="ggregory" type="fix" due-to="DaGeRe, Gary
Gregory">
+ IOUtils.write() throws OutOfMemoryError/NegativeArraySizeException
while writing big strings #343.
+ </action>
<!-- ADD -->
<action issue="IO-726" dev="ggregory" type="fix" due-to="shollander,
Gary Gregory">
Add MemoryMappedFileInputStream #215.
diff --git a/src/main/java/org/apache/commons/io/IOUtils.java
b/src/main/java/org/apache/commons/io/IOUtils.java
index 44cf6eb4..518ba579 100644
--- a/src/main/java/org/apache/commons/io/IOUtils.java
+++ b/src/main/java/org/apache/commons/io/IOUtils.java
@@ -40,6 +40,7 @@ import java.net.URL;
import java.net.URLConnection;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
+import java.nio.channels.Channels;
import java.nio.channels.ReadableByteChannel;
import java.nio.channels.Selector;
import java.nio.charset.Charset;
@@ -968,10 +969,7 @@ public class IOUtils {
*/
public static int copy(final InputStream inputStream, final OutputStream
outputStream) throws IOException {
final long count = copyLarge(inputStream, outputStream);
- if (count > Integer.MAX_VALUE) {
- return EOF;
- }
- return (int) count;
+ return count > Integer.MAX_VALUE ? EOF : (int) count;
}
/**
@@ -3360,9 +3358,13 @@ public class IOUtils {
* @throws IOException if an I/O error occurs
* @since 2.3
*/
+ @SuppressWarnings("resource")
public static void write(final String data, final OutputStream output,
final Charset charset) throws IOException {
if (data != null) {
- output.write(data.getBytes(Charsets.toCharset(charset)));
+ // Use Charset#encode(String), since calling
String#getBytes(Charset) might result in
+ // NegativeArraySizeException or OutOfMemoryError.
+ // The underlying OutputStream should not be closed, so the
channel is not closed.
+
Channels.newChannel(output).write(Charsets.toCharset(charset).encode(data));
}
}
diff --git a/src/test/java/org/apache/commons/io/IOUtilsTest.java
b/src/test/java/org/apache/commons/io/IOUtilsTest.java
index 207bc2dc..71a406ca 100644
--- a/src/test/java/org/apache/commons/io/IOUtilsTest.java
+++ b/src/test/java/org/apache/commons/io/IOUtilsTest.java
@@ -292,39 +292,6 @@ public class IOUtilsTest {
}
}
- /**
- * IO-764 IOUtils.write() throws NegativeArraySizeException while writing
big strings.
- * <pre>
- * java.lang.OutOfMemoryError: Java heap space
- * at java.lang.StringCoding.encode(StringCoding.java:350)
- * at java.lang.String.getBytes(String.java:941)
- * at org.apache.commons.io.IOUtils.write(IOUtils.java:3367)
- * at
org.apache.commons.io.IOUtilsTest.testBigString(IOUtilsTest.java:1659)
- * </pre>
- */
- @Test
- public void testBigString() throws IOException {
- // 3_000_000 is a size that we can allocate for the test string with
Java 8 on the command line as:
- // mvn clean test -Dtest=IOUtilsTest -DtestBigString=3000000
- // 6_000_000 failed with the above
- //
- // TODO Can we mock the test string for this test to pretend to be
larger?
- // Mocking the length seems simple but how about the data?
- final int repeat = Integer.getInteger("testBigString", 3_000_000);
- final String data;
- try {
- data = StringUtils.repeat("\uD83D", repeat);
- } catch (OutOfMemoryError e) {
- System.err.printf("Don't fail the test if we cannot build the
fixture, just log, fixture size = %,d%n.", repeat);
- e.printStackTrace();
- return;
- }
- try (CountingOutputStream os = new
CountingOutputStream(NullOutputStream.INSTANCE)) {
- IOUtils.write(data, os, StandardCharsets.UTF_8);
- assertEquals(repeat, os.getByteCount());
- }
- }
-
@Test
public void testClose() {
assertDoesNotThrow(() -> IOUtils.close((Closeable) null));
@@ -1673,4 +1640,49 @@ public class IOUtilsTest {
testToString_URL(null);
}
+ /**
+ * IO-764 IOUtils.write() throws NegativeArraySizeException while writing
big strings.
+ * <pre>
+ * java.lang.OutOfMemoryError: Java heap space
+ * at java.lang.StringCoding.encode(StringCoding.java:350)
+ * at java.lang.String.getBytes(String.java:941)
+ * at org.apache.commons.io.IOUtils.write(IOUtils.java:3367)
+ * at
org.apache.commons.io.IOUtilsTest.testBigString(IOUtilsTest.java:1659)
+ * </pre>
+ */
+ @Test
+ public void testWriteBigString() throws IOException {
+ // 3_000_000 is a size that we can allocate for the test string with
Java 8 on the command line as:
+ // mvn clean test -Dtest=IOUtilsTest -DtestBigString=3000000
+ // 6_000_000 failed with the above
+ //
+ // TODO Can we mock the test string for this test to pretend to be
larger?
+ // Mocking the length seems simple but how about the data?
+ final int repeat = Integer.getInteger("testBigString", 3_000_000);
+ final String data;
+ try {
+ data = StringUtils.repeat("\uD83D", repeat);
+ } catch (OutOfMemoryError e) {
+ System.err.printf("Don't fail the test if we cannot build the
fixture, just log, fixture size = %,d%n.", repeat);
+ e.printStackTrace();
+ return;
+ }
+ try (CountingOutputStream os = new
CountingOutputStream(NullOutputStream.INSTANCE)) {
+ IOUtils.write(data, os, StandardCharsets.UTF_8);
+ assertEquals(repeat, os.getByteCount());
+ }
+ }
+
+ @Test
+ public void testWriteLittleString() throws IOException {
+ final String data = "\uD83D";
+ // White-box test to check that not closing the internal channel is
not a problem.
+ for (int i = 0; i < 1_000_000; i++) {
+ try (CountingOutputStream os = new
CountingOutputStream(NullOutputStream.INSTANCE)) {
+ IOUtils.write(data, os, StandardCharsets.UTF_8);
+ assertEquals(data.length(), os.getByteCount());
+ }
+ }
+ }
+
}
