This is an automated email from the ASF dual-hosted git repository.
bharat pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 7097e5b HADOOP-17905. Modify Text.ensureCapacity() to efficiently max
out the… (#3423)
7097e5b is described below
commit 7097e5b793de68880ace8413f3be3ccd9d6d7e3c
Author: pbacsko <[email protected]>
AuthorDate: Thu Sep 30 02:25:29 2021 +0200
HADOOP-17905. Modify Text.ensureCapacity() to efficiently max out the…
(#3423)
---
.../src/main/java/org/apache/hadoop/io/Text.java | 16 +++++++++++++---
1 file changed, 13 insertions(+), 3 deletions(-)
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
index f39b1b7..5ca7f3c 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/Text.java
@@ -34,7 +34,6 @@ import java.text.StringCharacterIterator;
import java.util.Arrays;
import org.apache.avro.reflect.Stringable;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -73,6 +72,10 @@ public class Text extends BinaryComparable
}
};
+ // max size of the byte array, seems to be a safe choice for multiple JVMs
+ // (see ArrayList.MAX_ARRAY_SIZE)
+ private static final int ARRAY_MAX_SIZE = Integer.MAX_VALUE - 8;
+
private static final byte[] EMPTY_BYTES = new byte[0];
private byte[] bytes = EMPTY_BYTES;
@@ -302,8 +305,15 @@ public class Text extends BinaryComparable
private boolean ensureCapacity(final int capacity) {
if (bytes.length < capacity) {
// Try to expand the backing array by the factor of 1.5x
- // (by taking the current size + diving it by half)
- int targetSize = Math.max(capacity, bytes.length + (bytes.length >> 1));
+ // (by taking the current size + diving it by half).
+ //
+ // If the calculated value is beyond the size
+ // limit, we cap it to ARRAY_MAX_SIZE
+
+ long targetSizeLong = bytes.length + (bytes.length >> 1);
+ int targetSize = (int)Math.min(targetSizeLong, ARRAY_MAX_SIZE);
+ targetSize = Math.max(capacity, targetSize);
+
bytes = new byte[targetSize];
return true;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
