[hadoop] 01/02: Revert "HDFS-15202 Boost short circuit cache (rebase PR-1884) (#2016)"
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch branch-3.3
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 289f8acc644ff8f30452bec35e2833af57994112
Author: Wei-Chiu Chuang
AuthorDate: Mon May 18 09:41:03 2020 -0700
Revert "HDFS-15202 Boost short circuit cache (rebase PR-1884) (#2016)"
This reverts commit ad9a6a0ee3d6cb3bde5e23c73151c0857d47ffd4.
---
.../java/org/apache/hadoop/hdfs/ClientContext.java | 20 +--
.../hadoop/hdfs/client/HdfsClientConfigKeys.java | 2 -
.../hdfs/client/impl/BlockReaderFactory.java | 9 +-
.../hadoop/hdfs/client/impl/DfsClientConf.java | 21 +--
.../src/main/resources/hdfs-default.xml| 10 --
.../hadoop/fs/TestEnhancedByteBufferAccess.java| 6 +-
.../hdfs/client/impl/TestBlockReaderFactory.java | 14 +-
.../hdfs/client/impl/TestBlockReaderLocal.java | 190 ++---
.../hdfs/shortcircuit/TestShortCircuitCache.java | 19 +--
9 files changed, 76 insertions(+), 215 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
index 7a03240..cbd941b 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
@@ -77,7 +77,7 @@ public class ClientContext {
/**
* Caches short-circuit file descriptors, mmap regions.
*/
- private final ShortCircuitCache[] shortCircuitCache;
+ private final ShortCircuitCache shortCircuitCache;
/**
* Caches TCP and UNIX domain sockets for reuse.
@@ -132,23 +132,13 @@ public class ClientContext {
*/
private DeadNodeDetector deadNodeDetector = null;
- /**
- * ShortCircuitCache array size.
- */
- private final int clientShortCircuitNum;
-
private ClientContext(String name, DfsClientConf conf,
Configuration config) {
final ShortCircuitConf scConf = conf.getShortCircuitConf();
this.name = name;
this.confString = scConf.confAsString();
-this.clientShortCircuitNum = conf.getClientShortCircuitNum();
-this.shortCircuitCache = new ShortCircuitCache[this.clientShortCircuitNum];
-for (int i = 0; i < this.clientShortCircuitNum; i++) {
- this.shortCircuitCache[i] = ShortCircuitCache.fromConf(scConf);
-}
-
+this.shortCircuitCache = ShortCircuitCache.fromConf(scConf);
this.peerCache = new PeerCache(scConf.getSocketCacheCapacity(),
scConf.getSocketCacheExpiry());
this.keyProviderCache = new KeyProviderCache(
@@ -238,11 +228,7 @@ public class ClientContext {
}
public ShortCircuitCache getShortCircuitCache() {
-return shortCircuitCache[0];
- }
-
- public ShortCircuitCache getShortCircuitCache(long idx) {
-return shortCircuitCache[(int) (idx % clientShortCircuitNum)];
+return shortCircuitCache;
}
public PeerCache getPeerCache() {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index 0c35c8d..efc2766 100755
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -144,8 +144,6 @@ public interface HdfsClientConfigKeys {
"dfs.short.circuit.shared.memory.watcher.interrupt.check.ms";
int DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT =
6;
- String DFS_CLIENT_SHORT_CIRCUIT_NUM = "dfs.client.short.circuit.num";
- int DFS_CLIENT_SHORT_CIRCUIT_NUM_DEFAULT = 1;
String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY =
"dfs.client.slow.io.warning.threshold.ms";
longDFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 3;
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
index 028d629..a3b611c 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
@@ -476,8 +476,7 @@ public class BlockReaderFactory implements
ShortCircuitReplicaCreator {
"giving up on BlockReaderLocal.", this, pathInfo);
return null;
}
-ShortCircuitCache cache =
-clientContext.getShortCircuitCache(block.getBlockId());
+ShortCircuitCache cache = clientContext.getShortCircuitCache();
ExtendedBlockId
[hadoop] 01/02: Revert "HDFS-15202 Boost short circuit cache (rebase PR-1884) (#2016)"
This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 4525292d41482330a86f1cc3935e072f9f67c308
Author: Wei-Chiu Chuang
AuthorDate: Mon May 18 09:22:05 2020 -0700
Revert "HDFS-15202 Boost short circuit cache (rebase PR-1884) (#2016)"
This reverts commit 86e6aa8eec538e142044e2b6415ec1caff5e9cbd.
---
.../java/org/apache/hadoop/hdfs/ClientContext.java | 20 +--
.../hadoop/hdfs/client/HdfsClientConfigKeys.java | 2 -
.../hdfs/client/impl/BlockReaderFactory.java | 9 +-
.../hadoop/hdfs/client/impl/DfsClientConf.java | 21 +--
.../src/main/resources/hdfs-default.xml| 10 --
.../hadoop/fs/TestEnhancedByteBufferAccess.java| 6 +-
.../hdfs/client/impl/TestBlockReaderFactory.java | 14 +-
.../hdfs/client/impl/TestBlockReaderLocal.java | 190 ++---
.../hdfs/shortcircuit/TestShortCircuitCache.java | 19 +--
9 files changed, 76 insertions(+), 215 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
index 7a03240..cbd941b 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
@@ -77,7 +77,7 @@ public class ClientContext {
/**
* Caches short-circuit file descriptors, mmap regions.
*/
- private final ShortCircuitCache[] shortCircuitCache;
+ private final ShortCircuitCache shortCircuitCache;
/**
* Caches TCP and UNIX domain sockets for reuse.
@@ -132,23 +132,13 @@ public class ClientContext {
*/
private DeadNodeDetector deadNodeDetector = null;
- /**
- * ShortCircuitCache array size.
- */
- private final int clientShortCircuitNum;
-
private ClientContext(String name, DfsClientConf conf,
Configuration config) {
final ShortCircuitConf scConf = conf.getShortCircuitConf();
this.name = name;
this.confString = scConf.confAsString();
-this.clientShortCircuitNum = conf.getClientShortCircuitNum();
-this.shortCircuitCache = new ShortCircuitCache[this.clientShortCircuitNum];
-for (int i = 0; i < this.clientShortCircuitNum; i++) {
- this.shortCircuitCache[i] = ShortCircuitCache.fromConf(scConf);
-}
-
+this.shortCircuitCache = ShortCircuitCache.fromConf(scConf);
this.peerCache = new PeerCache(scConf.getSocketCacheCapacity(),
scConf.getSocketCacheExpiry());
this.keyProviderCache = new KeyProviderCache(
@@ -238,11 +228,7 @@ public class ClientContext {
}
public ShortCircuitCache getShortCircuitCache() {
-return shortCircuitCache[0];
- }
-
- public ShortCircuitCache getShortCircuitCache(long idx) {
-return shortCircuitCache[(int) (idx % clientShortCircuitNum)];
+return shortCircuitCache;
}
public PeerCache getPeerCache() {
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index e8b5402..ab3f6f2 100755
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -144,8 +144,6 @@ public interface HdfsClientConfigKeys {
"dfs.short.circuit.shared.memory.watcher.interrupt.check.ms";
int DFS_SHORT_CIRCUIT_SHARED_MEMORY_WATCHER_INTERRUPT_CHECK_MS_DEFAULT =
6;
- String DFS_CLIENT_SHORT_CIRCUIT_NUM = "dfs.client.short.circuit.num";
- int DFS_CLIENT_SHORT_CIRCUIT_NUM_DEFAULT = 1;
String DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY =
"dfs.client.slow.io.warning.threshold.ms";
longDFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT = 3;
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
index 028d629..a3b611c 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/BlockReaderFactory.java
@@ -476,8 +476,7 @@ public class BlockReaderFactory implements
ShortCircuitReplicaCreator {
"giving up on BlockReaderLocal.", this, pathInfo);
return null;
}
-ShortCircuitCache cache =
-clientContext.getShortCircuitCache(block.getBlockId());
+ShortCircuitCache cache = clientContext.getShortCircuitCache();
ExtendedBlockId key =
