Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 6475d468f -> 019770c76
HDFS-4946. Allow preferLocalNode in BlockPlacementPolicyDefault to be
configurable. Contributed by James Kinley and Nathan Roberts.
(cherry picked from commit ba6d5ed955aa34cdcb7aaa4b1025da6ee5e91355)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/019770c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/019770c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/019770c7
Branch: refs/heads/branch-2.7
Commit: 019770c7691e5bb69afcea39e41dfbf5aa038913
Parents: 6475d46
Author: Kihwal Lee <[email protected]>
Authored: Thu Feb 18 14:10:43 2016 -0600
Committer: Kihwal Lee <[email protected]>
Committed: Thu Feb 18 14:10:43 2016 -0600
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 5 ++++-
.../blockmanagement/BlockPlacementPolicyDefault.java | 7 ++++++-
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 12 ++++++++++++
4 files changed, 25 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/019770c7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ff9a371..2ff5bc8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -30,6 +30,9 @@ Release 2.7.3 - UNRELEASED
HDFS-9669. TcpPeerServer should respect ipc.server.listen.queue.size
(Elliot Clark via cmccabe)
+ HDFS-4946. Allow preferLocalNode in BlockPlacementPolicyDefault to be
+ configurable (James Kinley and Nathan Roberts via kihwal)
+
OPTIMIZATIONS
HDFS-8845. DiskChecker should not traverse the entire tree (Chang Li via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/019770c7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index c2a5296..9fc68f8 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -621,7 +621,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys
{
public static final String DFS_WEB_AUTHENTICATION_KERBEROS_KEYTAB_KEY =
"dfs.web.authentication.kerberos.keytab";
public static final String DFS_NAMENODE_MAX_OP_SIZE_KEY =
"dfs.namenode.max.op.size";
public static final int DFS_NAMENODE_MAX_OP_SIZE_DEFAULT = 50 * 1024 *
1024;
-
+ public static final String
DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_KEY =
+ "dfs.namenode.block-placement-policy.default.prefer-local-node";
+ public static final boolean
DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_DEFAULT = true;
+
public static final String DFS_BLOCK_LOCAL_PATH_ACCESS_USER_KEY =
"dfs.block.local-path-access.user";
public static final String DFS_DOMAIN_SOCKET_PATH_KEY =
"dfs.domain.socket.path";
public static final String DFS_DOMAIN_SOCKET_PATH_DEFAULT = "";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/019770c7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 1ed3d7d..d39c68e 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -61,7 +61,7 @@ public class BlockPlacementPolicyDefault extends
BlockPlacementPolicy {
};
protected boolean considerLoad;
- private boolean preferLocalNode = true;
+ private boolean preferLocalNode;
protected NetworkTopology clusterMap;
protected Host2NodesMap host2datanodeMap;
private FSClusterStats stats;
@@ -94,6 +94,11 @@ public class BlockPlacementPolicyDefault extends
BlockPlacementPolicy {
this.staleInterval = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_KEY,
DFSConfigKeys.DFS_NAMENODE_STALE_DATANODE_INTERVAL_DEFAULT);
+ this.preferLocalNode = conf.getBoolean(
+ DFSConfigKeys.
+ DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_KEY,
+ DFSConfigKeys.
+
DFS_NAMENODE_BLOCKPLACEMENTPOLICY_DEFAULT_PREFER_LOCAL_NODE_DEFAULT);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/019770c7/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index cee0d3c..0ef4bd3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -798,6 +798,18 @@
</property>
<property>
+ <name>dfs.namenode.block-placement-policy.default.prefer-local-node</name>
+ <value>true</value>
+ <description>Controls how the default block placement policy places
+ the first replica of a block. When true, it will prefer the node where
+ the client is running. When false, it will prefer a node in the same rack
+ as the client. Setting to false avoids situations where entire copies of
+ large files end up on a single node, thus creating hotspots.
+ </description>
+</property>
+
+
+<property>
<name>dfs.stream-buffer-size</name>
<value>4096</value>
<description>The size of buffer to stream files.
