[37/50] [abbrv] hadoop git commit: HDFS-13166: [SPS]: Implement caching mechanism to keep LIVE datanodes to minimize costly getLiveDatanodeStorageReport() calls. Contributed by Rakesh R.
HDFS-13166: [SPS]: Implement caching mechanism to keep LIVE datanodes to
minimize costly getLiveDatanodeStorageReport() calls. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75ccc139
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75ccc139
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75ccc139
Branch: refs/heads/trunk
Commit: 75ccc1396b67cdc0d4992a4af3911f9f88c2
Parents: 8467ec2
Author: Surendra Singh Lilhore
Authored: Thu Mar 1 00:08:37 2018 +0530
Committer: Uma Maheswara Rao Gangumalla
Committed: Sun Aug 12 03:06:05 2018 -0700
--
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +
.../NamenodeProtocolServerSideTranslatorPB.java | 19 --
.../NamenodeProtocolTranslatorPB.java | 17 -
.../hdfs/server/namenode/NameNodeRpcServer.java | 13 -
.../hdfs/server/namenode/sps/Context.java | 24 +-
.../namenode/sps/DatanodeCacheManager.java | 121 +++
.../namenode/sps/IntraSPSNameNodeContext.java | 23 +-
.../namenode/sps/StoragePolicySatisfier.java| 340 ++-
.../hdfs/server/protocol/NamenodeProtocol.java | 16 -
.../hdfs/server/sps/ExternalSPSContext.java | 32 +-
.../src/main/proto/NamenodeProtocol.proto | 25 --
.../src/main/resources/hdfs-default.xml | 11 +
.../src/site/markdown/ArchivalStorage.md| 2 +-
.../TestStoragePolicySatisfyWorker.java | 3 +
.../TestPersistentStoragePolicySatisfier.java | 6 +
.../TestStoragePolicySatisfierWithHA.java | 3 +
.../sps/TestStoragePolicySatisfier.java | 4 +
...stStoragePolicySatisfierWithStripedFile.java | 24 +-
.../TestStoragePolicySatisfyAdminCommands.java | 3 +
19 files changed, 431 insertions(+), 260 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75ccc139/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b354d64..cf383d0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -646,6 +646,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys
{
public static final String DFS_SPS_MAX_OUTSTANDING_PATHS_KEY =
"dfs.storage.policy.satisfier.max.outstanding.paths";
public static final int DFS_SPS_MAX_OUTSTANDING_PATHS_DEFAULT = 1;
+ // SPS datanode cache config, defaulting to 5mins.
+ public static final String DFS_SPS_DATANODE_CACHE_REFRESH_INTERVAL_MS =
+ "dfs.storage.policy.satisfier.datanode.cache.refresh.interval.ms";
+ public static final long DFS_SPS_DATANODE_CACHE_REFRESH_INTERVAL_MS_DEFAULT =
+ 30L;
// SPS keytab configurations, by default it is disabled.
public static final String DFS_SPS_ADDRESS_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75ccc139/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index ed176cc..e4283c6 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -23,8 +23,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder;
import
org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.VersionRequestProto;
import
org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.VersionResponseProto;
-import
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.CheckDNSpaceRequestProto;
-import
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.CheckDNSpaceResponseProto;
import
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointRequestProto;
import
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointResponseProto;
import
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.ErrorReportRequestProto;
@@ -277,21 +275,4 @@ public class NamenodeProtocolServerSideTranslatorPB
[37/50] [abbrv] hadoop git commit: HDFS-13166: [SPS]: Implement caching mechanism to keep LIVE datanodes to minimize costly getLiveDatanodeStorageReport() calls. Contributed by Rakesh R.
HDFS-13166: [SPS]: Implement caching mechanism to keep LIVE datanodes to
minimize costly getLiveDatanodeStorageReport() calls. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75ccc139
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75ccc139
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75ccc139
Branch: refs/heads/HDFS-10285
Commit: 75ccc1396b67cdc0d4992a4af3911f9f88c2
Parents: 8467ec2
Author: Surendra Singh Lilhore
Authored: Thu Mar 1 00:08:37 2018 +0530
Committer: Uma Maheswara Rao Gangumalla
Committed: Sun Aug 12 03:06:05 2018 -0700
--
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 +
.../NamenodeProtocolServerSideTranslatorPB.java | 19 --
.../NamenodeProtocolTranslatorPB.java | 17 -
.../hdfs/server/namenode/NameNodeRpcServer.java | 13 -
.../hdfs/server/namenode/sps/Context.java | 24 +-
.../namenode/sps/DatanodeCacheManager.java | 121 +++
.../namenode/sps/IntraSPSNameNodeContext.java | 23 +-
.../namenode/sps/StoragePolicySatisfier.java| 340 ++-
.../hdfs/server/protocol/NamenodeProtocol.java | 16 -
.../hdfs/server/sps/ExternalSPSContext.java | 32 +-
.../src/main/proto/NamenodeProtocol.proto | 25 --
.../src/main/resources/hdfs-default.xml | 11 +
.../src/site/markdown/ArchivalStorage.md| 2 +-
.../TestStoragePolicySatisfyWorker.java | 3 +
.../TestPersistentStoragePolicySatisfier.java | 6 +
.../TestStoragePolicySatisfierWithHA.java | 3 +
.../sps/TestStoragePolicySatisfier.java | 4 +
...stStoragePolicySatisfierWithStripedFile.java | 24 +-
.../TestStoragePolicySatisfyAdminCommands.java | 3 +
19 files changed, 431 insertions(+), 260 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75ccc139/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index b354d64..cf383d0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -646,6 +646,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys
{
public static final String DFS_SPS_MAX_OUTSTANDING_PATHS_KEY =
"dfs.storage.policy.satisfier.max.outstanding.paths";
public static final int DFS_SPS_MAX_OUTSTANDING_PATHS_DEFAULT = 1;
+ // SPS datanode cache config, defaulting to 5mins.
+ public static final String DFS_SPS_DATANODE_CACHE_REFRESH_INTERVAL_MS =
+ "dfs.storage.policy.satisfier.datanode.cache.refresh.interval.ms";
+ public static final long DFS_SPS_DATANODE_CACHE_REFRESH_INTERVAL_MS_DEFAULT =
+ 30L;
// SPS keytab configurations, by default it is disabled.
public static final String DFS_SPS_ADDRESS_KEY =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75ccc139/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index ed176cc..e4283c6 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -23,8 +23,6 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo.DatanodeInfoBuilder;
import
org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.VersionRequestProto;
import
org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.VersionResponseProto;
-import
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.CheckDNSpaceRequestProto;
-import
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.CheckDNSpaceResponseProto;
import
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointRequestProto;
import
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointResponseProto;
import
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.ErrorReportRequestProto;
@@ -277,21 +275,4 @@ public class
