This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-2.8
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.8 by this push:
new daffd46 HDFS-14535. The default 8KB buffer in
requestFileDescriptors#BufferedOutputStream is causing lots of heap allocation
in HBase when using short-circut read
daffd46 is described below
commit daffd466b3ae1efd0be1bf59fd4584edb0a9ac37
Author: huzheng <[email protected]>
AuthorDate: Tue Jun 4 10:54:46 2019 +0800
HDFS-14535. The default 8KB buffer in
requestFileDescriptors#BufferedOutputStream is causing lots of heap allocation
in HBase when using short-circut read
(cherry picked from commit ea3b0a184405c3feca024a560807ea215b6858b9)
(cherry picked from commit 9b61ecfcbeeb1d3c320f775b43c68d3e39a271a3)
(cherry picked from commit b082628e5a39fc1bc9d6094beb9c253cc096c701)
(cherry picked from commit 2b9c4fccf4a8930bfff99ab4928b9b3b4980d3b2)
(cherry picked from commit 8b90c69357cf515661ec8a08e61ddd03e884b3ab)
(cherry picked from commit d14fd913f46fc8e794c39f4cc44d1674aa156ee0)
---
.../java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
index 5486295..8e22b9e 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
@@ -86,6 +86,7 @@ import org.slf4j.LoggerFactory;
@InterfaceAudience.Private
public class BlockReaderFactory implements ShortCircuitReplicaCreator {
static final Logger LOG = LoggerFactory.getLogger(BlockReaderFactory.class);
+ private static final int SMALL_BUFFER_SIZE = 512;
public static class FailureInjector {
public void injectRequestFileDescriptorsFailure() throws IOException {
@@ -589,7 +590,7 @@ public class BlockReaderFactory implements
ShortCircuitReplicaCreator {
Slot slot) throws IOException {
ShortCircuitCache cache = clientContext.getShortCircuitCache();
final DataOutputStream out =
- new DataOutputStream(new BufferedOutputStream(peer.getOutputStream()));
+ new DataOutputStream(new BufferedOutputStream(peer.getOutputStream(),
SMALL_BUFFER_SIZE));
SlotId slotId = slot == null ? null : slot.getSlotId();
new Sender(out).requestShortCircuitFds(block, token, slotId, 1,
failureInjector.getSupportsReceiptVerification());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
