subject:"\[2\/2\] hadoop git commit\: HDFS\-11799. Introduce a config to allow setting up write pipeline with fewer nodes than replication factor. Contributed by Brahma Reddy Battula"

[2/2] hadoop git commit: HDFS-11799. Introduce a config to allow setting up write pipeline with fewer nodes than replication factor. Contributed by Brahma Reddy Battula

2017-09-19 Thread brahma

HDFS-11799. Introduce a config to allow setting up write pipeline with fewer 
nodes than replication factor. Contributed by Brahma Reddy Battula

(cherry picked from commit fda1221c55101d97ac62e1ee4e3ddf9a915d5363)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a81167e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a81167e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a81167e2

Branch: refs/heads/branch-2.8
Commit: a81167e2ec9c8eff13b1fb69b11c0265987e8ce4
Parents: 67ada5d
Author: Brahma Reddy Battula 
Authored: Tue Sep 19 11:25:45 2017 +0530
Committer: Brahma Reddy Battula 
Committed: Tue Sep 19 11:34:52 2017 +0530

--
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  13 +-
 .../org/apache/hadoop/hdfs/DataStreamer.java|  31 +-
 .../hdfs/client/HdfsClientConfigKeys.java   |   2 +
 .../src/main/resources/hdfs-default.xml |  17 ++
 .../TestReplaceDatanodeFailureReplication.java  | 291 +++
 .../hadoop/tools/TestHdfsConfigFields.java  |   5 +-
 6 files changed, 355 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a81167e2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 1e999d2..9cfec3f 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -231,6 +231,7 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
   final SocketFactory socketFactory;
   final ReplaceDatanodeOnFailure dtpReplaceDatanodeOnFailure;
   final FileSystem.Statistics stats;
+  final short dtpReplaceDatanodeOnFailureReplication;
   private final URI namenodeUri;
   private final Random r = new Random();
   private SocketAddress[] localInterfaceAddrs;
@@ -311,7 +312,17 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
 this.socketFactory = NetUtils.getSocketFactory(conf, ClientProtocol.class);
 this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf);
 this.smallBufferSize = DFSUtilClient.getSmallBufferSize(conf);
-
+this.dtpReplaceDatanodeOnFailureReplication = (short) conf
+.getInt(HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.
+MIN_REPLICATION,
+HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.
+MIN_REPLICATION_DEFAULT);
+if (LOG.isDebugEnabled()) {
+  LOG.debug(
+  "Sets " + HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.
+  MIN_REPLICATION + " to "
+  + dtpReplaceDatanodeOnFailureReplication);
+}
 this.ugi = UserGroupInformation.getCurrentUser();
 
 this.namenodeUri = nameNodeUri;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a81167e2/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 879c4ab..fb3f1d1 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -1348,7 +1348,36 @@ class DataStreamer extends Daemon {
   setPipeline(lb);
 
   //find the new datanode
-  final int d = findNewDatanode(original);
+  final int d;
+  try {
+d = findNewDatanode(original);
+  } catch (IOException ioe) {
+// check the minimal number of nodes available to decide whether to
+// continue the write.
+
+//if live block location datanodes is greater than or equal to
+// HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.
+// MIN_REPLICATION threshold value, continue writing to the
+// remaining nodes. Otherwise throw exception.
+//
+// If HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.
+// MIN_REPLICATION is set to 0 or less than zero, an exception will be
+// thrown if a replacement could not be found.
+
+if (dfsClient.dtpReplaceDatanodeOnFailureReplication > 0 && 
nodes.length
+>=

[2/2] hadoop git commit: HDFS-11799. Introduce a config to allow setting up write pipeline with fewer nodes than replication factor. Contributed by Brahma Reddy Battula

2017-09-19 Thread brahma

HDFS-11799. Introduce a config to allow setting up write pipeline with fewer 
nodes than replication factor. Contributed by Brahma Reddy Battula

(cherry picked from commit fda1221c55101d97ac62e1ee4e3ddf9a915d5363)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ce365e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ce365e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ce365e9

Branch: refs/heads/branch-3.0
Commit: 1ce365e91e742943098e1843d5b15e7e9e51ef72
Parents: 6ee79d5
Author: Brahma Reddy Battula 
Authored: Tue Sep 19 11:25:45 2017 +0530
Committer: Brahma Reddy Battula 
Committed: Tue Sep 19 11:27:41 2017 +0530

--
 .../java/org/apache/hadoop/hdfs/DFSClient.java  |  13 +-
 .../org/apache/hadoop/hdfs/DataStreamer.java|  31 +-
 .../hdfs/client/HdfsClientConfigKeys.java   |   2 +
 .../src/main/resources/hdfs-default.xml |  17 ++
 .../TestReplaceDatanodeFailureReplication.java  | 291 +++
 .../hadoop/tools/TestHdfsConfigFields.java  |   4 +-
 6 files changed, 354 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ce365e9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 772049d..7e8e95b 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -223,6 +223,7 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
   final String clientName;
   final SocketFactory socketFactory;
   final ReplaceDatanodeOnFailure dtpReplaceDatanodeOnFailure;
+  final short dtpReplaceDatanodeOnFailureReplication;
   private final FileSystem.Statistics stats;
   private final URI namenodeUri;
   private final Random r = new Random();
@@ -305,7 +306,17 @@ public class DFSClient implements java.io.Closeable, 
RemotePeerFactory,
 this.socketFactory = NetUtils.getSocketFactory(conf, ClientProtocol.class);
 this.dtpReplaceDatanodeOnFailure = ReplaceDatanodeOnFailure.get(conf);
 this.smallBufferSize = DFSUtilClient.getSmallBufferSize(conf);
-
+this.dtpReplaceDatanodeOnFailureReplication = (short) conf
+.getInt(HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.
+MIN_REPLICATION,
+HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.
+MIN_REPLICATION_DEFAULT);
+if (LOG.isDebugEnabled()) {
+  LOG.debug(
+  "Sets " + HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.
+  MIN_REPLICATION + " to "
+  + dtpReplaceDatanodeOnFailureReplication);
+}
 this.ugi = UserGroupInformation.getCurrentUser();
 
 this.namenodeUri = nameNodeUri;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ce365e9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 4eafca1..99fa5f3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -1384,7 +1384,36 @@ class DataStreamer extends Daemon {
   setPipeline(lb);
 
   //find the new datanode
-  final int d = findNewDatanode(original);
+  final int d;
+  try {
+d = findNewDatanode(original);
+  } catch (IOException ioe) {
+// check the minimal number of nodes available to decide whether to
+// continue the write.
+
+//if live block location datanodes is greater than or equal to
+// HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.
+// MIN_REPLICATION threshold value, continue writing to the
+// remaining nodes. Otherwise throw exception.
+//
+// If HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.
+// MIN_REPLICATION is set to 0 or less than zero, an exception will be
+// thrown if a replacement could not be found.
+
+if (dfsClient.dtpReplaceDatanodeOnFailureReplication > 0 && 
nodes.length
+>= dfsClient.dtpReplaceDatanodeOnFailureReplication) {
+

[2/2] hadoop git commit: HDFS-11799. Introduce a config to allow setting up write pipeline with fewer nodes than replication factor. Contributed by Brahma Reddy Battula

[2/2] hadoop git commit: HDFS-11799. Introduce a config to allow setting up write pipeline with fewer nodes than replication factor. Contributed by Brahma Reddy Battula

2 matches

Site Navigation

Mail list logo

Footer information