hadoop git commit: HADOOP-11819. HttpServerFunctionalTest#prepareTestWebapp should create web app directory if it does not exist. (Contributed by Rohith)

[vinayakumarb]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 d77705c98 - a75679cdb


HADOOP-11819. HttpServerFunctionalTest#prepareTestWebapp should create web app 
directory if it does not exist. (Contributed by Rohith)

(cherry picked from commit b5a0b24643688270e77f85aaf1af24fee2c556cc)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a75679cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a75679cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a75679cd

Branch: refs/heads/branch-2
Commit: a75679cdb33e07fb982bac02fbe295192a0eb78a
Parents: d77705c
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Apr 14 11:58:12 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Apr 14 11:59:07 2015 +0530

--
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../apache/hadoop/http/HttpServerFunctionalTest.java| 12 +++-
 2 files changed, 10 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a75679cd/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 742a054..c77bbb7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -50,6 +50,9 @@ Release 2.8.0 - UNRELEASED
 HADOOP-7713. dfs -count -q should label output column (Jonathan Allen
 via aw)
 
+HADOOP-11819. HttpServerFunctionalTest#prepareTestWebapp should create web
+app directory if it does not exist. (Rohith via vinayakumarb)
+
   OPTIMIZATIONS
 
 HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a75679cd/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
index 4a4de41..faa27a7 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
@@ -123,11 +123,13 @@ public class HttpServerFunctionalTest extends Assert {
 File testWebappDir = new File(webapps +
 File.separatorChar + TEST);
 try {
-if (!testWebappDir.exists()) {
-  fail(Test webapp dir  + testWebappDir.getCanonicalPath() +  missing);
-}
-}
-catch (IOException e) {
+  if (!testWebappDir.exists()) {
+if (!testWebappDir.mkdirs()) {
+  fail(Test webapp dir  + testWebappDir.getCanonicalPath()
+  +  can not be created);
+}
+  }
+} catch (IOException e) {
 }
   }
 

hadoop git commit: HADOOP-11819. HttpServerFunctionalTest#prepareTestWebapp should create web app directory if it does not exist. (Contributed by Rohith)

[vinayakumarb]

Repository: hadoop
Updated Branches:
  refs/heads/trunk d4e8c0959 - b5a0b2464


HADOOP-11819. HttpServerFunctionalTest#prepareTestWebapp should create web app 
directory if it does not exist. (Contributed by Rohith)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b5a0b246
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b5a0b246
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b5a0b246

Branch: refs/heads/trunk
Commit: b5a0b24643688270e77f85aaf1af24fee2c556cc
Parents: d4e8c09
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Apr 14 11:58:12 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Apr 14 11:58:12 2015 +0530

--
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../apache/hadoop/http/HttpServerFunctionalTest.java| 12 +++-
 2 files changed, 10 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5a0b246/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 68913bc..bb2eca5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -494,6 +494,9 @@ Release 2.8.0 - UNRELEASED
 HADOOP-7713. dfs -count -q should label output column (Jonathan Allen
 via aw)
 
+HADOOP-11819. HttpServerFunctionalTest#prepareTestWebapp should create web
+app directory if it does not exist. (Rohith via vinayakumarb)
+
   OPTIMIZATIONS
 
 HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b5a0b246/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
index 4a4de41..faa27a7 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/HttpServerFunctionalTest.java
@@ -123,11 +123,13 @@ public class HttpServerFunctionalTest extends Assert {
 File testWebappDir = new File(webapps +
 File.separatorChar + TEST);
 try {
-if (!testWebappDir.exists()) {
-  fail(Test webapp dir  + testWebappDir.getCanonicalPath() +  missing);
-}
-}
-catch (IOException e) {
+  if (!testWebappDir.exists()) {
+if (!testWebappDir.mkdirs()) {
+  fail(Test webapp dir  + testWebappDir.getCanonicalPath()
+  +  can not be created);
+}
+  }
+} catch (IOException e) {
 }
   }
 

hadoop git commit: HADOOP-9642. Configuration to resolve environment variables via ${env.VARIABLE} references (Kengo Seki via aw)

[aw]

Repository: hadoop
Updated Branches:
  refs/heads/trunk b9b832a0d - d4e8c0959


HADOOP-9642. Configuration to resolve environment variables via ${env.VARIABLE} 
references (Kengo Seki via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4e8c095
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4e8c095
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4e8c095

Branch: refs/heads/trunk
Commit: d4e8c09593b8edc4d0494d64ce19ac78c6f31277
Parents: b9b832a
Author: Allen Wittenauer a...@apache.org
Authored: Tue Apr 14 08:20:13 2015 +0200
Committer: Allen Wittenauer a...@apache.org
Committed: Tue Apr 14 08:20:13 2015 +0200

--
 hadoop-common-project/hadoop-common/CHANGES.txt |  6 +-
 .../org/apache/hadoop/conf/Configuration.java   | 55 +-
 .../apache/hadoop/conf/TestConfiguration.java   | 77 ++--
 3 files changed, 110 insertions(+), 28 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4e8c095/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0fb6e92..68913bc 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -25,7 +25,8 @@ Trunk (Unreleased)
 
   NEW FEATURES
 
-HADOOP-6590. Add a username check for hadoop sub-commands (John Smith via 
aw)
+HADOOP-6590. Add a username check for hadoop sub-commands (John Smith via
+aw)
 
 HADOOP-11353. Add support for .hadooprc (aw)
 
@@ -41,6 +42,9 @@ Trunk (Unreleased)
 
 HADOOP-11565. Add --slaves shell option (aw)
 
+HADOOP-9642. Configuration to resolve environment variables via
+${env.VARIABLE} references (Kengo Seki via aw)
+
   IMPROVEMENTS
 
 HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4e8c095/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 8a312ff..7c25e6c 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -146,6 +146,8 @@ import com.google.common.base.Preconditions;
  * available properties are:ol
  * liOther properties defined in this Configuration; and, if a name is
  * undefined here,/li
+ * liEnvironment variables in {@link System#getenv()} if a name starts with
+ * env., or/li
  * liProperties in {@link System#getProperties()}./li
  * /ol
  *
@@ -160,13 +162,25 @@ import com.google.common.base.Preconditions;
  *  lt;propertygt;
  *lt;namegt;tempdirlt;/namegt;
  *lt;valuegt;${ibasedir/i}/tmplt;/valuegt;
- *  lt;/propertygt;/pre/tt
+ *  lt;/propertygt;
+ *
+ *  lt;propertygt;
+ *lt;namegt;otherdirlt;/namegt;
+ *lt;valuegt;${ienv.BASE_DIR/i}/otherlt;/valuegt;
+ *  lt;/propertygt;
+ *  /pre/tt
  *
- * When ttconf.get(tempdir)/tt is called, then tt${ibasedir/i}/tt
+ * pWhen ttconf.get(tempdir)/tt is called, then 
tt${ibasedir/i}/tt
  * will be resolved to another property in this Configuration, while
  * tt${iuser.name/i}/tt would then ordinarily be resolved to the value
  * of the System property with that name.
- * By default, warnings will be given to any deprecated configuration 
+ * pWhen ttconf.get(otherdir)/tt is called, then 
tt${ienv.BASE_DIR/i}/tt
+ * will be resolved to the value of the tt${iBASE_DIR/i}/tt 
environment variable.
+ * It supports tt${ienv.NAME:-default/i}/tt and 
tt${ienv.NAME-default/i}/tt notations.
+ * The former is resolved to “default” if tt${iNAME/i}/tt 
environment variable is undefined
+ * or its value is empty.
+ * The latter behaves the same way only if tt${iNAME/i}/tt is 
undefined.
+ * pBy default, warnings will be given to any deprecated configuration 
  * parameters and these are suppressible by configuring
  * ttlog4j.logger.org.apache.hadoop.conf.Configuration.deprecation/tt in
  * log4j.properties file.
@@ -915,6 +929,7 @@ public class Configuration implements 
IterableMap.EntryString,String,
* Attempts to repeatedly expand the value {@code expr} by replacing the
* left-most substring of the form ${var} in the following precedence order
* ol
+   *   liby the value of the environment variable var if defined/li
*   liby the value of the Java system property var if defined/li
*   liby 

[1/2] hadoop git commit: HDFS-6666. Abort NameNode and DataNode startup if security is enabled but block access token is not enabled. Contributed by Vijay Bhat.

[cnauroth]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 a75679cdb - 9779b79ed
  refs/heads/trunk b5a0b2464 - d45aa7647


HDFS-. Abort NameNode and DataNode startup if security is enabled but block 
access token is not enabled. Contributed by Vijay Bhat.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d45aa764
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d45aa764
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d45aa764

Branch: refs/heads/trunk
Commit: d45aa7647b1fecf81860ec7b563085be2af99a0b
Parents: b5a0b24
Author: cnauroth cnaur...@apache.org
Authored: Tue Apr 14 09:59:01 2015 -0700
Committer: cnauroth cnaur...@apache.org
Committed: Tue Apr 14 09:59:01 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 +
 .../server/blockmanagement/BlockManager.java|  7 +-
 .../hadoop/hdfs/server/datanode/DataNode.java   | 14 +++
 .../hdfs/server/namenode/FSNamesystem.java  | 10 ++-
 .../sasl/SaslDataTransferTestCase.java  | 41 +++--
 .../server/namenode/TestSecureNameNode.java | 90 
 6 files changed, 115 insertions(+), 50 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d45aa764/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 7414d33..f9b27da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -494,6 +494,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-7701. Support reporting per storage type quota and usage
 with hadoop/hdfs shell. (Peter Shi via Arpit Agarwal)
 
+HDFS-. Abort NameNode and DataNode startup if security is enabled but
+block access token is not enabled. (Vijay Bhat via cnauroth)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d45aa764/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 8540dc1..4c9582c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -359,7 +359,7 @@ public class BlockManager {
   }
 
   private static BlockTokenSecretManager createBlockTokenSecretManager(
-  final Configuration conf) {
+  final Configuration conf) throws IOException {
 final boolean isEnabled = conf.getBoolean(
 DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, 
 DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT);
@@ -367,10 +367,11 @@ public class BlockManager {
 
 if (!isEnabled) {
   if (UserGroupInformation.isSecurityEnabled()) {
-LOG.error(Security is enabled but block access tokens  +
+String errMessage = Security is enabled but block access tokens  +
 (via  + DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY + )  +
 aren't enabled. This may cause issues  +
-when clients attempt to talk to a DataNode.);
+when clients attempt to connect to a DataNode. Aborting NameNode;
+throw new IOException(errMessage);
   }
   return null;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d45aa764/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index b0e79e3..cb22b73 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1183,6 +1183,20 @@ public class DataNode extends ReconfigurableBase
 if (!UserGroupInformation.isSecurityEnabled()) {
   return;
 }
+
+// Abort out of inconsistent state if Kerberos is enabled
+// but block access tokens are not enabled.
+boolean isEnabled = conf.getBoolean(
+DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY,
+DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT);
+

[Hadoop Wiki] Update of HadoopStreaming by EricMoyer

[Apache Wiki]

Dear Wiki user,

You have subscribed to a wiki page or wiki category on Hadoop Wiki for change 
notification.

The HadoopStreaming page has been changed by EricMoyer:
https://wiki.apache.org/hadoop/HadoopStreaming?action=diffrev1=14rev2=15

Comment:
Fix hadoop streaming link

  == See Also ==
  * HowToDebugMapReducePrograms
  * HadoopStreaming/AlternativeInterfaces
- * [[http://hadoop.apache.org/mapreduce/docs/current/streaming.html|Hadoop 
Streaming]]
+ * 
[[http://hadoop.apache.org/docs/current/hadoop-mapreduce-client/hadoop-mapreduce-client-core/HadoopStreaming.html|Hadoop
 Streaming]]
  

[2/2] hadoop git commit: HDFS-6666. Abort NameNode and DataNode startup if security is enabled but block access token is not enabled. Contributed by Vijay Bhat.

[cnauroth]

HDFS-. Abort NameNode and DataNode startup if security is enabled but block 
access token is not enabled. Contributed by Vijay Bhat.

(cherry picked from commit d45aa7647b1fecf81860ec7b563085be2af99a0b)

Conflicts:

hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestSecureNameNode.java


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9779b79e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9779b79e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9779b79e

Branch: refs/heads/branch-2
Commit: 9779b79ed8371cbfecf77e020236e1f751e8efeb
Parents: a75679c
Author: cnauroth cnaur...@apache.org
Authored: Tue Apr 14 09:59:01 2015 -0700
Committer: cnauroth cnaur...@apache.org
Committed: Tue Apr 14 10:03:53 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../server/blockmanagement/BlockManager.java|  7 ++--
 .../hadoop/hdfs/server/datanode/DataNode.java   | 14 +++
 .../hdfs/server/namenode/FSNamesystem.java  | 10 +++--
 .../sasl/SaslDataTransferTestCase.java  | 41 
 5 files changed, 60 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9779b79e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2c2603b..d47de9d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -173,6 +173,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-7701. Support reporting per storage type quota and usage
 with hadoop/hdfs shell. (Peter Shi via Arpit Agarwal)
 
+HDFS-. Abort NameNode and DataNode startup if security is enabled but
+block access token is not enabled. (Vijay Bhat via cnauroth)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9779b79e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index f9097b8..ae13a29 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -359,7 +359,7 @@ public class BlockManager {
   }
 
   private static BlockTokenSecretManager createBlockTokenSecretManager(
-  final Configuration conf) {
+  final Configuration conf) throws IOException {
 final boolean isEnabled = conf.getBoolean(
 DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, 
 DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_DEFAULT);
@@ -367,10 +367,11 @@ public class BlockManager {
 
 if (!isEnabled) {
   if (UserGroupInformation.isSecurityEnabled()) {
-LOG.error(Security is enabled but block access tokens  +
+String errMessage = Security is enabled but block access tokens  +
 (via  + DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY + )  +
 aren't enabled. This may cause issues  +
-when clients attempt to talk to a DataNode.);
+when clients attempt to connect to a DataNode. Aborting NameNode;
+throw new IOException(errMessage);
   }
   return null;
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9779b79e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 1b3c4ff..eddbddb 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1190,6 +1190,20 @@ public class DataNode extends ReconfigurableBase
 if (!UserGroupInformation.isSecurityEnabled()) {
   return;
 }
+
+// Abort out of inconsistent state if Kerberos is enabled
+// but block access tokens are not enabled.
+boolean isEnabled = conf.getBoolean(
+DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY,
+

[2/2] hadoop git commit: HDFS-8055. NullPointerException when topology script is missing. Contributed by Anu Engineer.

[cnauroth]

HDFS-8055. NullPointerException when topology script is missing. Contributed by 
Anu Engineer.

(cherry picked from commit fef596df038112cbbc86c4dc49314e274fca0190)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7ebecff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7ebecff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7ebecff

Branch: refs/heads/branch-2
Commit: c7ebecfff5b609a66914b40f95294ad87a2afcc7
Parents: 9779b79
Author: cnauroth cnaur...@apache.org
Authored: Tue Apr 14 10:19:30 2015 -0700
Committer: cnauroth cnaur...@apache.org
Committed: Tue Apr 14 10:19:45 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 +
 .../server/blockmanagement/DatanodeManager.java | 14 ++-
 .../blockmanagement/TestDatanodeManager.java| 94 +++-
 .../test/resources/topology-broken-script.cmd   | 22 +
 .../test/resources/topology-broken-script.sh| 23 +
 .../src/test/resources/topology-script.cmd  | 18 
 .../src/test/resources/topology-script.sh   | 21 +
 7 files changed, 172 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ebecff/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d47de9d..3fec3b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -176,6 +176,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-. Abort NameNode and DataNode startup if security is enabled but
 block access token is not enabled. (Vijay Bhat via cnauroth)
 
+HDFS-8055. NullPointerException when topology script is missing.
+(Anu Engineer via cnauroth)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ebecff/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index d7e0721..808ee75 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -372,9 +372,17 @@ public class DatanodeManager {
 if (client == null) {
   ListString hosts = new ArrayListString (1);
   hosts.add(targethost);
-  String rName = dnsToSwitchMapping.resolve(hosts).get(0);
-  if (rName != null)
-client = new NodeBase(rName + NodeBase.PATH_SEPARATOR_STR + 
targethost);
+  ListString resolvedHosts = dnsToSwitchMapping.resolve(hosts);
+  if (resolvedHosts != null  !resolvedHosts.isEmpty()) {
+String rName = resolvedHosts.get(0);
+if (rName != null) {
+  client = new NodeBase(rName + NodeBase.PATH_SEPARATOR_STR +
+targethost);
+}
+  } else {
+LOG.error(Node Resolution failed. Please make sure that rack  +
+  awareness scripts are functional.);
+  }
 }
 
 ComparatorDatanodeInfo comparator = avoidStaleDataNodesForRead ?

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ebecff/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
index a4a6263..bf167a5 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
@@ -19,6 +19,10 @@
 package org.apache.hadoop.hdfs.server.blockmanagement;
 
 import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -31,6 +35,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import 

hadoop git commit: YARN-3266. RMContext#inactiveNodes should have NodeId as map key. Contributed by Chengbing Liu

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk fef596df0 - b46ee1e7a


YARN-3266. RMContext#inactiveNodes should have NodeId as map key. Contributed 
by Chengbing Liu


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b46ee1e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b46ee1e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b46ee1e7

Branch: refs/heads/trunk
Commit: b46ee1e7a31007985b88072d9af3d97c33a261a7
Parents: fef596d
Author: Jian He jia...@apache.org
Authored: Tue Apr 14 10:54:22 2015 -0700
Committer: Jian He jia...@apache.org
Committed: Tue Apr 14 10:54:22 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |  3 ++
 .../resourcemanager/RMActiveServiceContext.java |  6 +--
 .../yarn/server/resourcemanager/RMContext.java  |  2 +-
 .../server/resourcemanager/RMContextImpl.java   |  2 +-
 .../resourcemanager/rmnode/RMNodeImpl.java  | 10 ++--
 .../resourcemanager/webapp/RMWebServices.java   |  2 +-
 .../resourcemanager/TestRMNodeTransitions.java  | 53 ++--
 .../resourcemanager/webapp/TestRMWebApp.java|  6 +--
 .../webapp/TestRMWebServicesNodes.java  |  9 ++--
 9 files changed, 72 insertions(+), 21 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ee1e7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bec5ce1..4961ae0 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -192,6 +192,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3472. Fixed possible leak in DelegationTokenRenewer#allTokens.
 (Rohith Sharmaks via jianhe)
 
+YARN-3266. RMContext#inactiveNodes should have NodeId as map key.
+(Chengbing Liu via jianhe)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ee1e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 03fc40e..e4c72d8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -68,8 +68,8 @@ public class RMActiveServiceContext {
   private final ConcurrentMapNodeId, RMNode nodes =
   new ConcurrentHashMapNodeId, RMNode();
 
-  private final ConcurrentMapString, RMNode inactiveNodes =
-  new ConcurrentHashMapString, RMNode();
+  private final ConcurrentMapNodeId, RMNode inactiveNodes =
+  new ConcurrentHashMapNodeId, RMNode();
 
   private final ConcurrentMapApplicationId, ByteBuffer systemCredentials =
   new ConcurrentHashMapApplicationId, ByteBuffer();
@@ -185,7 +185,7 @@ public class RMActiveServiceContext {
 
   @Private
   @Unstable
-  public ConcurrentMapString, RMNode getInactiveRMNodes() {
+  public ConcurrentMapNodeId, RMNode getInactiveRMNodes() {
 return this.inactiveNodes;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b46ee1e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index ecf6166..bc50268 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -61,7 +61,7 @@ public interface RMContext {
   
   ConcurrentMapApplicationId, ByteBuffer 

[Hadoop Wiki] Trivial Update of ContributorsGroup by QwertyManiac

[Apache Wiki]

Dear Wiki user,

You have subscribed to a wiki page or wiki category on Hadoop Wiki for change 
notification.

The ContributorsGroup page has been changed by QwertyManiac:
https://wiki.apache.org/hadoop/ContributorsGroup?action=diffrev1=101rev2=102

   * ErikBakker
   * Erik Gorset
   * EricHwang
+  * EricMoyer
   * EricYang
   * ermanpattuk
   * EstebanMolinaEstolano

hadoop git commit: YARN-3462. Patches applied for YARN-2424 are inconsistent between trunk and branch-2. Contributed by Naganarasimha G R.

[harsh]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 38b031d6b - 01af29106


YARN-3462. Patches applied for YARN-2424 are inconsistent between trunk and 
branch-2. Contributed by Naganarasimha G R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01af2910
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01af2910
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01af2910

Branch: refs/heads/branch-2
Commit: 01af29106a1603eff9f4e622a6919d49cb0bfa65
Parents: 38b031d
Author: Harsh J ha...@cloudera.com
Authored: Wed Apr 15 09:41:43 2015 +0530
Committer: Harsh J ha...@cloudera.com
Committed: Wed Apr 15 09:41:43 2015 +0530

--
 hadoop-yarn-project/CHANGES.txt |  3 +++
 .../src/main/resources/yarn-default.xml | 21 ++--
 .../nodemanager/LinuxContainerExecutor.java | 12 +++
 3 files changed, 22 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/01af2910/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f77aafd..b5e850e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -165,6 +165,9 @@ Release 2.7.1 - UNRELEASED
 
   BUG FIXES
 
+YARN-3462. Patches applied for YARN-2424 are inconsistent between
+trunk and branch-2. (Naganarasimha G R via harsh)
+
 Release 2.7.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/01af2910/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index a469cae..66400c8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1036,21 +1036,22 @@
   /property
 
   property
-descriptionThis determines which of the two modes that LCE should use on 
a non-secure
-cluster.  If this value is set to true, then all containers will be 
launched as the user 
-specified in 
yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user.  If 
-this value is set to false, then containers will run as the user who 
submitted the 
-application.
-/description
+descriptionThis determines which of the two modes that LCE should use on
+  a non-secure cluster.  If this value is set to true, then all containers
+  will be launched as the user specified in
+  yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user.  If
+  this value is set to false, then containers will run as the user who
+  submitted the application./description
 
nameyarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users/name
 valuetrue/value
   /property
 
   property
-descriptionThe UNIX user that containers will run as when 
Linux-container-executor
-is used in nonsecure mode (a use case for this is using cgroups) if the
-yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users is 
set 
-to true./description
+descriptionThe UNIX user that containers will run as when
+  Linux-container-executor is used in nonsecure mode (a use case for this
+  is using cgroups) if the
+  yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users is
+  set to true./description
 
nameyarn.nodemanager.linux-container-executor.nonsecure-mode.local-user/name
 valuenobody/value
   /property

http://git-wip-us.apache.org/repos/asf/hadoop/blob/01af2910/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 59b35ce..fac71d3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ 

hadoop git commit: YARN-3462. Patches applied for YARN-2424 are inconsistent between trunk and branch-2. Contributed by Naganarasimha G R.

[harsh]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 50c8d0631 - d4a462c02


YARN-3462. Patches applied for YARN-2424 are inconsistent between trunk and 
branch-2. Contributed by Naganarasimha G R.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d4a462c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d4a462c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d4a462c0

Branch: refs/heads/branch-2.7
Commit: d4a462c02e9be1e082ef4f9c62a47cf93a5c9b7c
Parents: 50c8d06
Author: Harsh J ha...@cloudera.com
Authored: Wed Apr 15 09:41:43 2015 +0530
Committer: Harsh J ha...@cloudera.com
Committed: Wed Apr 15 09:42:20 2015 +0530

--
 hadoop-yarn-project/CHANGES.txt |  3 +++
 .../src/main/resources/yarn-default.xml | 21 ++--
 .../nodemanager/LinuxContainerExecutor.java | 12 +++
 3 files changed, 22 insertions(+), 14 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4a462c0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5e44bc3..9072edf 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -12,6 +12,9 @@ Release 2.7.1 - UNRELEASED
 
   BUG FIXES
 
+YARN-3462. Patches applied for YARN-2424 are inconsistent between
+trunk and branch-2. (Naganarasimha G R via harsh)
+
 Release 2.7.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4a462c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index a469cae..66400c8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1036,21 +1036,22 @@
   /property
 
   property
-descriptionThis determines which of the two modes that LCE should use on 
a non-secure
-cluster.  If this value is set to true, then all containers will be 
launched as the user 
-specified in 
yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user.  If 
-this value is set to false, then containers will run as the user who 
submitted the 
-application.
-/description
+descriptionThis determines which of the two modes that LCE should use on
+  a non-secure cluster.  If this value is set to true, then all containers
+  will be launched as the user specified in
+  yarn.nodemanager.linux-container-executor.nonsecure-mode.local-user.  If
+  this value is set to false, then containers will run as the user who
+  submitted the application./description
 
nameyarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users/name
 valuetrue/value
   /property
 
   property
-descriptionThe UNIX user that containers will run as when 
Linux-container-executor
-is used in nonsecure mode (a use case for this is using cgroups) if the
-yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users is 
set 
-to true./description
+descriptionThe UNIX user that containers will run as when
+  Linux-container-executor is used in nonsecure mode (a use case for this
+  is using cgroups) if the
+  yarn.nodemanager.linux-container-executor.nonsecure-mode.limit-users is
+  set to true./description
 
nameyarn.nodemanager.linux-container-executor.nonsecure-mode.local-user/name
 valuenobody/value
   /property

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d4a462c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 59b35ce..fac71d3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ 

[2/3] hadoop git commit: YARN-3361. CapacityScheduler side changes to support non-exclusive node labels. Contributed by Wangda Tan

[jianhe]

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fefda64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 882498a..eb64d43 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -56,8 +56,6 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManage
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.AssignmentInformation;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
 import org.apache.hadoop.yarn.util.resource.Resources;
@@ -377,16 +375,29 @@ public class ParentQueue extends AbstractCSQueue {
 
   @Override
   public synchronized CSAssignment assignContainers(Resource clusterResource,
-  FiCaSchedulerNode node, ResourceLimits resourceLimits) {
-CSAssignment assignment = 
-new CSAssignment(Resources.createResource(0, 0), NodeType.NODE_LOCAL);
-SetString nodeLabels = node.getLabels();
-
+  FiCaSchedulerNode node, ResourceLimits resourceLimits,
+  SchedulingMode schedulingMode) {
 // if our queue cannot access this node, just return
-if (!SchedulerUtils.checkQueueAccessToNode(accessibleLabels, nodeLabels)) {
-  return assignment;
+if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY
+ !accessibleToPartition(node.getPartition())) {
+  return NULL_ASSIGNMENT;
+}
+
+// Check if this queue need more resource, simply skip allocation if this
+// queue doesn't need more resources.
+if (!super.hasPendingResourceRequest(node.getPartition(),
+clusterResource, schedulingMode)) {
+  if (LOG.isDebugEnabled()) {
+LOG.debug(Skip this queue= + getQueuePath()
++ , because it doesn't need more resource, schedulingMode=
++ schedulingMode.name() +  node-partition= + 
node.getPartition());
+  }
+  return NULL_ASSIGNMENT;
 }
 
+CSAssignment assignment = 
+new CSAssignment(Resources.createResource(0, 0), NodeType.NODE_LOCAL);
+
 while (canAssign(clusterResource, node)) {
   if (LOG.isDebugEnabled()) {
 LOG.debug(Trying to assign containers to child-queue of 
@@ -396,15 +407,17 @@ public class ParentQueue extends AbstractCSQueue {
   // Are we over maximum-capacity for this queue?
   // This will also consider parent's limits and also continuous 
reservation
   // looking
-  if (!super.canAssignToThisQueue(clusterResource, nodeLabels, 
resourceLimits,
-  minimumAllocation, Resources.createResource(getMetrics()
-  .getReservedMB(), getMetrics().getReservedVirtualCores( {
+  if (!super.canAssignToThisQueue(clusterResource, node.getPartition(),
+  resourceLimits, minimumAllocation, Resources.createResource(
+  getMetrics().getReservedMB(), getMetrics()
+  .getReservedVirtualCores()), schedulingMode)) {
 break;
   }
   
   // Schedule
-  CSAssignment assignedToChild = 
-  assignContainersToChildQueues(clusterResource, node, resourceLimits);
+  CSAssignment assignedToChild =
+  assignContainersToChildQueues(clusterResource, node, resourceLimits,
+  schedulingMode);
   assignment.setType(assignedToChild.getType());
   
   // Done if no child-queue assigned anything
@@ -413,7 +426,7 @@ public class ParentQueue extends AbstractCSQueue {
   assignedToChild.getResource(), Resources.none())) {
 // Track resource utilization for the parent-queue
 super.allocateResource(clusterResource, assignedToChild.getResource(),
-nodeLabels);
+

hadoop git commit: YARN-3436. Fix URIs in documantion of YARN web service REST APIs. Contributed by Bibin A Chundatt.

[ozawa]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 9ebbf1bfc - 3daca4f04


YARN-3436. Fix URIs in documantion of YARN web service REST APIs. Contributed 
by Bibin A Chundatt.

(cherry picked from commit 05007b45e58bd9052f503cfb8c17bcfd22a686e3)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3daca4f0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3daca4f0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3daca4f0

Branch: refs/heads/branch-2
Commit: 3daca4f04070c639dfcafa2e14d90a3f0af112c4
Parents: 9ebbf1b
Author: Tsuyoshi Ozawa oz...@apache.org
Authored: Tue Apr 14 14:13:55 2015 -0500
Committer: Tsuyoshi Ozawa oz...@apache.org
Committed: Tue Apr 14 14:14:17 2015 -0500

--
 hadoop-yarn-project/CHANGES.txt| 3 +++
 .../hadoop-yarn-site/src/site/markdown/WebServicesIntro.md | 6 +++---
 2 files changed, 6 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3daca4f0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 059c5a3..f77aafd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -150,6 +150,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3266. RMContext#inactiveNodes should have NodeId as map key.
 (Chengbing Liu via jianhe)
 
+YARN-3436. Fix URIs in documantion of YARN web service REST APIs.
+(Bibin A Chundatt via ozawa)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3daca4f0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md
index 0e89a50..0a396d3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md
@@ -97,7 +97,7 @@ After calling an HTTP request, an application should check 
the response status c
 
  JSON response with single resource
 
-HTTP Request: GET 
http://rmhost.domain:8088/ws/v1/cluster/app/application\_1324057493980\_0001
+HTTP Request: GET 
http://rmhost.domain:8088/ws/v1/cluster/apps/application\_1324057493980\_0001
 
 Response Status Line: HTTP/1.1 200 OK
 
@@ -137,7 +137,7 @@ Response Body:
 
 Here we request information about an application that doesn't exist yet.
 
-HTTP Request: GET 
http://rmhost.domain:8088/ws/v1/cluster/app/application\_1324057493980\_
+HTTP Request: GET 
http://rmhost.domain:8088/ws/v1/cluster/apps/application\_1324057493980\_
 
 Response Status Line: HTTP/1.1 404 Not Found
 
@@ -566,4 +566,4 @@ Output:
   queue : a1
}
 }
-```
\ No newline at end of file
+```

hadoop git commit: YARN-3436. Fix URIs in documantion of YARN web service REST APIs. Contributed by Bibin A Chundatt.

[ozawa]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 0fefda645 - 05007b45e


YARN-3436. Fix URIs in documantion of YARN web service REST APIs. Contributed 
by Bibin A Chundatt.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/05007b45
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/05007b45
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/05007b45

Branch: refs/heads/trunk
Commit: 05007b45e58bd9052f503cfb8c17bcfd22a686e3
Parents: 0fefda6
Author: Tsuyoshi Ozawa oz...@apache.org
Authored: Tue Apr 14 14:13:55 2015 -0500
Committer: Tsuyoshi Ozawa oz...@apache.org
Committed: Tue Apr 14 14:13:55 2015 -0500

--
 hadoop-yarn-project/CHANGES.txt| 3 +++
 .../hadoop-yarn-site/src/site/markdown/WebServicesIntro.md | 6 +++---
 2 files changed, 6 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/05007b45/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2933c79..2db02a2a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -198,6 +198,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3266. RMContext#inactiveNodes should have NodeId as map key.
 (Chengbing Liu via jianhe)
 
+YARN-3436. Fix URIs in documantion of YARN web service REST APIs.
+(Bibin A Chundatt via ozawa)
+
 Release 2.7.1 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/05007b45/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md
index 0e89a50..0a396d3 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WebServicesIntro.md
@@ -97,7 +97,7 @@ After calling an HTTP request, an application should check 
the response status c
 
  JSON response with single resource
 
-HTTP Request: GET 
http://rmhost.domain:8088/ws/v1/cluster/app/application\_1324057493980\_0001
+HTTP Request: GET 
http://rmhost.domain:8088/ws/v1/cluster/apps/application\_1324057493980\_0001
 
 Response Status Line: HTTP/1.1 200 OK
 
@@ -137,7 +137,7 @@ Response Body:
 
 Here we request information about an application that doesn't exist yet.
 
-HTTP Request: GET 
http://rmhost.domain:8088/ws/v1/cluster/app/application\_1324057493980\_
+HTTP Request: GET 
http://rmhost.domain:8088/ws/v1/cluster/apps/application\_1324057493980\_
 
 Response Status Line: HTTP/1.1 404 Not Found
 
@@ -566,4 +566,4 @@ Output:
   queue : a1
}
 }
-```
\ No newline at end of file
+```

[3/3] hadoop git commit: YARN-3361. CapacityScheduler side changes to support non-exclusive node labels. Contributed by Wangda Tan

[jianhe]

YARN-3361. CapacityScheduler side changes to support non-exclusive node labels. 
Contributed by Wangda Tan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0fefda64
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0fefda64
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0fefda64

Branch: refs/heads/trunk
Commit: 0fefda645bca935b87b6bb8ca63e6f18340d59f5
Parents: b46ee1e
Author: Jian He jia...@apache.org
Authored: Tue Apr 14 11:36:37 2015 -0700
Committer: Jian He jia...@apache.org
Committed: Tue Apr 14 11:45:58 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |3 +
 .../hadoop/yarn/server/utils/BuilderUtils.java  |1 +
 .../rmapp/attempt/RMAppAttemptImpl.java |   10 +-
 .../scheduler/AppSchedulingInfo.java|   25 +-
 .../scheduler/ResourceUsage.java|8 +
 .../scheduler/SchedulerApplicationAttempt.java  |   44 +-
 .../scheduler/SchedulerUtils.java   |   87 +-
 .../scheduler/capacity/AbstractCSQueue.java |  243 +++--
 .../scheduler/capacity/CSQueue.java |5 +-
 .../scheduler/capacity/CapacityScheduler.java   |   91 +-
 .../CapacitySchedulerConfiguration.java |5 +
 .../scheduler/capacity/LeafQueue.java   |  368 ---
 .../scheduler/capacity/ParentQueue.java |   59 +-
 .../scheduler/capacity/SchedulingMode.java  |   44 +
 .../server/resourcemanager/Application.java |4 +
 .../yarn/server/resourcemanager/MockAM.java |8 +-
 .../yarn/server/resourcemanager/MockRM.java |   35 +-
 .../capacity/TestApplicationLimits.java |8 +-
 .../scheduler/capacity/TestChildQueueOrder.java |   41 +-
 .../capacity/TestContainerAllocation.java   |  390 +--
 .../scheduler/capacity/TestLeafQueue.java   |  148 +--
 .../TestNodeLabelContainerAllocation.java   | 1027 ++
 .../scheduler/capacity/TestParentQueue.java |  111 +-
 .../scheduler/capacity/TestReservations.java|  101 +-
 .../scheduler/capacity/TestUtils.java   |2 +
 25 files changed, 1914 insertions(+), 954 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fefda64/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4961ae0..2933c79 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -72,6 +72,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3443. Create a 'ResourceHandler' subsystem to ease addition of 
support 
 for new resource types on the NM. (Sidharta Seethana via junping_du)
 
+YARN-3361. CapacityScheduler side changes to support non-exclusive node
+labels. (Wangda Tan via jianhe)
+
   IMPROVEMENTS
 
 YARN-1880. Cleanup TestApplicationClientProtocolOnHA

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fefda64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 68d4ef9..f2146c8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -313,6 +313,7 @@ public class BuilderUtils {
 request.setResourceName(r.getResourceName());
 request.setCapability(r.getCapability());
 request.setNumContainers(r.getNumContainers());
+request.setNodeLabelExpression(r.getNodeLabelExpression());
 return request;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fefda64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 1be1727..1071831 100644

[1/3] hadoop git commit: YARN-3361. CapacityScheduler side changes to support non-exclusive node labels. Contributed by Wangda Tan

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/trunk b46ee1e7a - 0fefda645


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fefda64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
new file mode 100644
index 000..cf1b26f
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -0,0 +1,1027 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
+import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
+public class TestNodeLabelContainerAllocation {
+  private final int GB = 1024;
+
+  private YarnConfiguration conf;
+  
+  RMNodeLabelsManager mgr;
+
+  @Before
+  public void setUp() throws Exception {
+conf = new YarnConfiguration();
+conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+  ResourceScheduler.class);
+mgr = new NullRMNodeLabelsManager();
+mgr.init(conf);
+  }
+  
+  private Configuration getConfigurationWithQueueLabels(Configuration config) {
+CapacitySchedulerConfiguration conf =
+new CapacitySchedulerConfiguration(config);
+
+// Define top-level queues
+conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {a, 
b, c});
+conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, x, 100);
+conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, y, 100);
+
+final String A = CapacitySchedulerConfiguration.ROOT + .a;
+

[3/3] hadoop git commit: YARN-3361. CapacityScheduler side changes to support non-exclusive node labels. Contributed by Wangda Tan (cherry picked from commit 0fefda645bca935b87b6bb8ca63e6f18340d59f5)

[jianhe]

YARN-3361. CapacityScheduler side changes to support non-exclusive node labels. 
Contributed by Wangda Tan
(cherry picked from commit 0fefda645bca935b87b6bb8ca63e6f18340d59f5)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ebbf1bf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ebbf1bf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ebbf1bf

Branch: refs/heads/branch-2
Commit: 9ebbf1bfcea9942117727c08c6905dd444c230ae
Parents: 81bbee6
Author: Jian He jia...@apache.org
Authored: Tue Apr 14 11:36:37 2015 -0700
Committer: Jian He jia...@apache.org
Committed: Tue Apr 14 11:46:35 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |3 +
 .../hadoop/yarn/server/utils/BuilderUtils.java  |1 +
 .../rmapp/attempt/RMAppAttemptImpl.java |   10 +-
 .../scheduler/AppSchedulingInfo.java|   25 +-
 .../scheduler/ResourceUsage.java|8 +
 .../scheduler/SchedulerApplicationAttempt.java  |   44 +-
 .../scheduler/SchedulerUtils.java   |   87 +-
 .../scheduler/capacity/AbstractCSQueue.java |  243 +++--
 .../scheduler/capacity/CSQueue.java |5 +-
 .../scheduler/capacity/CapacityScheduler.java   |   91 +-
 .../CapacitySchedulerConfiguration.java |5 +
 .../scheduler/capacity/LeafQueue.java   |  368 ---
 .../scheduler/capacity/ParentQueue.java |   59 +-
 .../scheduler/capacity/SchedulingMode.java  |   44 +
 .../server/resourcemanager/Application.java |4 +
 .../yarn/server/resourcemanager/MockAM.java |8 +-
 .../yarn/server/resourcemanager/MockRM.java |   35 +-
 .../capacity/TestApplicationLimits.java |8 +-
 .../scheduler/capacity/TestChildQueueOrder.java |   41 +-
 .../capacity/TestContainerAllocation.java   |  390 +--
 .../scheduler/capacity/TestLeafQueue.java   |  148 +--
 .../TestNodeLabelContainerAllocation.java   | 1027 ++
 .../scheduler/capacity/TestParentQueue.java |  111 +-
 .../scheduler/capacity/TestReservations.java|  101 +-
 .../scheduler/capacity/TestUtils.java   |2 +
 25 files changed, 1914 insertions(+), 954 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ebbf1bf/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 478d0ae..059c5a3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -24,6 +24,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3443. Create a 'ResourceHandler' subsystem to ease addition of 
support 
 for new resource types on the NM. (Sidharta Seethana via junping_du)
 
+YARN-3361. CapacityScheduler side changes to support non-exclusive node
+labels. (Wangda Tan via jianhe)
+
   IMPROVEMENTS
 
 YARN-1880. Cleanup TestApplicationClientProtocolOnHA

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ebbf1bf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index 68d4ef9..f2146c8 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -313,6 +313,7 @@ public class BuilderUtils {
 request.setResourceName(r.getResourceName());
 request.setCapability(r.getCapability());
 request.setNumContainers(r.getNumContainers());
+request.setNodeLabelExpression(r.getNodeLabelExpression());
 return request;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ebbf1bf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
 

[1/3] hadoop git commit: YARN-3361. CapacityScheduler side changes to support non-exclusive node labels. Contributed by Wangda Tan (cherry picked from commit 0fefda645bca935b87b6bb8ca63e6f18340d59f5)

[jianhe]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 81bbee685 - 9ebbf1bfc


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ebbf1bf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
new file mode 100644
index 000..cf1b26f
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -0,0 +1,1027 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
+import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppReport;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
+public class TestNodeLabelContainerAllocation {
+  private final int GB = 1024;
+
+  private YarnConfiguration conf;
+  
+  RMNodeLabelsManager mgr;
+
+  @Before
+  public void setUp() throws Exception {
+conf = new YarnConfiguration();
+conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+  ResourceScheduler.class);
+mgr = new NullRMNodeLabelsManager();
+mgr.init(conf);
+  }
+  
+  private Configuration getConfigurationWithQueueLabels(Configuration config) {
+CapacitySchedulerConfiguration conf =
+new CapacitySchedulerConfiguration(config);
+
+// Define top-level queues
+conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {a, 
b, c});
+conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, x, 100);
+conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, y, 100);
+
+final String A = CapacitySchedulerConfiguration.ROOT + .a;
+

hadoop git commit: HDFS-8127. NameNode Failover during HA upgrade can cause DataNode to finalize upgrade. Contributed by Jing Zhao.

[jing9]

Repository: hadoop
Updated Branches:
  refs/heads/trunk 05007b45e - fddd55279


HDFS-8127. NameNode Failover during HA upgrade can cause DataNode to finalize 
upgrade. Contributed by Jing Zhao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fddd5527
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fddd5527
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fddd5527

Branch: refs/heads/trunk
Commit: fddd55279d0bdd08b3b40aba6fe2ded1d2e0d846
Parents: 05007b4
Author: Jing Zhao ji...@apache.org
Authored: Tue Apr 14 15:05:23 2015 -0700
Committer: Jing Zhao ji...@apache.org
Committed: Tue Apr 14 15:05:23 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |   3 +
 .../NamenodeProtocolServerSideTranslatorPB.java |  17 +++
 .../NamenodeProtocolTranslatorPB.java   |  16 +++
 .../hadoop/hdfs/server/namenode/FSImage.java|  15 +-
 .../hdfs/server/namenode/FSNamesystem.java  |   4 +-
 .../hadoop/hdfs/server/namenode/NNStorage.java  |   2 +-
 .../hdfs/server/namenode/NNUpgradeUtil.java |  52 ---
 .../hdfs/server/namenode/NameNodeRpcServer.java |   7 +
 .../server/namenode/ha/BootstrapStandby.java| 139 ---
 .../hdfs/server/protocol/NamenodeProtocol.java  |   7 +
 .../src/main/proto/NamenodeProtocol.proto   |  16 +++
 .../ha/TestBootstrapStandbyWithQJM.java |  89 +++-
 .../namenode/ha/TestDFSUpgradeWithHA.java   |   5 +-
 13 files changed, 321 insertions(+), 51 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fddd5527/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 274e9cb..690056d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -517,6 +517,9 @@ Release 2.7.1 - UNRELEASED
 
   BUG FIXES
 
+HDFS-8127. NameNode Failover during HA upgrade can cause DataNode to
+finalize upgrade. (jing9)
+
 Release 2.7.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fddd5527/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index 284808e..dbacded 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.VersionRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.VersionResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.ErrorReportRequestProto;
@@ -36,6 +37,8 @@ import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecen
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecentCheckpointTxIdResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdResponseProto;
+import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedRequestProto;
+import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RollEditLogRequestProto;
@@ -222,4 +225,18 @@ public class NamenodeProtocolServerSideTranslatorPB 
implements
 return VersionResponseProto.newBuilder()
 .setInfo(PBHelper.convert(info)).build();
   }
+
+  @Override
+  public IsUpgradeFinalizedResponseProto isUpgradeFinalized(
+  RpcController controller, IsUpgradeFinalizedRequestProto request)
+  

hadoop git commit: HDFS-8127. NameNode Failover during HA upgrade can cause DataNode to finalize upgrade. Contributed by Jing Zhao.

[jing9]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2 3daca4f04 - 38b031d6b


HDFS-8127. NameNode Failover during HA upgrade can cause DataNode to finalize 
upgrade. Contributed by Jing Zhao.

(cherry picked from commit fddd55279d0bdd08b3b40aba6fe2ded1d2e0d846)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38b031d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38b031d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38b031d6

Branch: refs/heads/branch-2
Commit: 38b031d6bab8527698bd186887d301bd6a63cf01
Parents: 3daca4f
Author: Jing Zhao ji...@apache.org
Authored: Tue Apr 14 15:05:23 2015 -0700
Committer: Jing Zhao ji...@apache.org
Committed: Tue Apr 14 15:12:37 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |   3 +
 .../NamenodeProtocolServerSideTranslatorPB.java |  17 +++
 .../NamenodeProtocolTranslatorPB.java   |  16 +++
 .../hadoop/hdfs/server/namenode/FSImage.java|  15 +-
 .../hdfs/server/namenode/FSNamesystem.java  |   4 +-
 .../hadoop/hdfs/server/namenode/NNStorage.java  |   2 +-
 .../hdfs/server/namenode/NNUpgradeUtil.java |  52 ---
 .../hdfs/server/namenode/NameNodeRpcServer.java |   7 +
 .../server/namenode/ha/BootstrapStandby.java| 141 ---
 .../hdfs/server/protocol/NamenodeProtocol.java  |   7 +
 .../src/main/proto/NamenodeProtocol.proto   |  16 +++
 .../ha/TestBootstrapStandbyWithQJM.java |  89 +++-
 .../namenode/ha/TestDFSUpgradeWithHA.java   |   5 +-
 13 files changed, 322 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/38b031d6/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3fec3b4..bf9a634 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -196,6 +196,9 @@ Release 2.7.1 - UNRELEASED
 
   BUG FIXES
 
+HDFS-8127. NameNode Failover during HA upgrade can cause DataNode to
+finalize upgrade. (jing9)
+
 Release 2.7.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/38b031d6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index 284808e..dbacded 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.VersionRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.VersionResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.ErrorReportRequestProto;
@@ -36,6 +37,8 @@ import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecen
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecentCheckpointTxIdResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdResponseProto;
+import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedRequestProto;
+import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RollEditLogRequestProto;
@@ -222,4 +225,18 @@ public class NamenodeProtocolServerSideTranslatorPB 
implements
 return VersionResponseProto.newBuilder()
 .setInfo(PBHelper.convert(info)).build();
   }
+
+  @Override
+  public IsUpgradeFinalizedResponseProto isUpgradeFinalized(
+

hadoop git commit: HDFS-8127. NameNode Failover during HA upgrade can cause DataNode to finalize upgrade. Contributed by Jing Zhao.

[jing9]

Repository: hadoop
Updated Branches:
  refs/heads/branch-2.7 a41c1c80b - 50c8d0631


HDFS-8127. NameNode Failover during HA upgrade can cause DataNode to finalize 
upgrade. Contributed by Jing Zhao.

(cherry picked from commit fddd55279d0bdd08b3b40aba6fe2ded1d2e0d846)
(cherry picked from commit 38b031d6bab8527698bd186887d301bd6a63cf01)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50c8d063
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50c8d063
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50c8d063

Branch: refs/heads/branch-2.7
Commit: 50c8d06314d41dc1030812f223a94e8893164274
Parents: a41c1c8
Author: Jing Zhao ji...@apache.org
Authored: Tue Apr 14 15:05:23 2015 -0700
Committer: Jing Zhao ji...@apache.org
Committed: Tue Apr 14 15:15:36 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |   3 +
 .../NamenodeProtocolServerSideTranslatorPB.java |  17 +++
 .../NamenodeProtocolTranslatorPB.java   |  16 +++
 .../hadoop/hdfs/server/namenode/FSImage.java|  15 +-
 .../hdfs/server/namenode/FSNamesystem.java  |   4 +-
 .../hadoop/hdfs/server/namenode/NNStorage.java  |   2 +-
 .../hdfs/server/namenode/NNUpgradeUtil.java |  52 ---
 .../hdfs/server/namenode/NameNodeRpcServer.java |   7 +
 .../server/namenode/ha/BootstrapStandby.java| 141 ---
 .../hdfs/server/protocol/NamenodeProtocol.java  |   7 +
 .../src/main/proto/NamenodeProtocol.proto   |  16 +++
 .../ha/TestBootstrapStandbyWithQJM.java |  89 +++-
 .../namenode/ha/TestDFSUpgradeWithHA.java   |   5 +-
 13 files changed, 322 insertions(+), 52 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/50c8d063/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1e1a0d8..7c9225a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -17,6 +17,9 @@ Release 2.7.1 - UNRELEASED
 
   BUG FIXES
 
+HDFS-8127. NameNode Failover during HA upgrade can cause DataNode to
+finalize upgrade. (jing9)
+
 Release 2.7.0 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/50c8d063/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
index 284808e..dbacded 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/NamenodeProtocolServerSideTranslatorPB.java
@@ -22,6 +22,7 @@ import java.io.IOException;
 import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.VersionRequestProto;
 import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.VersionResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.EndCheckpointResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.ErrorReportRequestProto;
@@ -36,6 +37,8 @@ import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecen
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetMostRecentCheckpointTxIdResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.GetTransactionIdResponseProto;
+import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedRequestProto;
+import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.IsUpgradeFinalizedResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterRequestProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RegisterResponseProto;
 import 
org.apache.hadoop.hdfs.protocol.proto.NamenodeProtocolProtos.RollEditLogRequestProto;
@@ -222,4 +225,18 @@ public class NamenodeProtocolServerSideTranslatorPB 
implements
 return VersionResponseProto.newBuilder()
 .setInfo(PBHelper.convert(info)).build();
   }
+
+