hadoop git commit: HDFS-8834. TestReplication is not valid after HDFS-6482. (Contributed by Lei Xu)

2015-07-28 Thread lei
Repository: hadoop
Updated Branches:
  refs/heads/trunk 975e138df -> f4f1b8b26


HDFS-8834. TestReplication is not valid after HDFS-6482. (Contributed by Lei Xu)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4f1b8b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4f1b8b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4f1b8b2

Branch: refs/heads/trunk
Commit: f4f1b8b267703b8bebab06e17e69a4a4de611592
Parents: 975e138
Author: Lei Xu 
Authored: Tue Jul 28 16:55:51 2015 -0700
Committer: Lei Xu 
Committed: Tue Jul 28 23:39:26 2015 -0700

--
 .../org/apache/hadoop/hdfs/TestReplication.java | 26 +++-
 1 file changed, 25 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4f1b8b2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
index b702da0..2139df9 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
@@ -18,6 +18,8 @@
 package org.apache.hadoop.hdfs;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
@@ -25,6 +27,10 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.io.RandomAccessFile;
 import java.net.InetSocketAddress;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.List;
@@ -505,12 +511,28 @@ public class TestReplication {
 if (data_dir.listFiles().length == 0) {
   nonParticipatedNodeDirs.add(data_dir);
 } else {
+  assertNull("participatedNodeDirs has already been set.",
+  participatedNodeDirs);
   participatedNodeDirs = data_dir;
 }
   }
+  assertEquals(2, nonParticipatedNodeDirs.size());
 
   String blockFile = null;
-  File[] listFiles = participatedNodeDirs.listFiles();
+  final List listFiles = new ArrayList<>();
+  Files.walkFileTree(participatedNodeDirs.toPath(),
+  new SimpleFileVisitor() {
+@Override
+public FileVisitResult visitFile(
+java.nio.file.Path file, BasicFileAttributes attrs)
+throws IOException {
+  listFiles.add(file.toFile());
+  return FileVisitResult.CONTINUE;
+}
+  }
+  );
+  assertFalse(listFiles.isEmpty());
+  int numReplicaCreated = 0;
   for (File file : listFiles) {
 if (file.getName().startsWith(Block.BLOCK_FILE_PREFIX)
 && !file.getName().endsWith("meta")) {
@@ -519,10 +541,12 @@ public class TestReplication {
 file1.mkdirs();
 new File(file1, blockFile).createNewFile();
 new File(file1, blockFile + "_1000.meta").createNewFile();
+numReplicaCreated++;
   }
   break;
 }
   }
+  assertEquals(2, numReplicaCreated);
 
   fs.setReplication(new Path("/test"), (short) 3);
   cluster.restartDataNodes(); // Lets detect all DNs about dummy copied



hadoop git commit: HDFS-8834. TestReplication is not valid after HDFS-6482. (Contributed by Lei Xu)

2015-07-28 Thread lei
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 dcf1f1fdf -> 9d8191604


HDFS-8834. TestReplication is not valid after HDFS-6482. (Contributed by Lei Xu)

(cherry picked from commit f4f1b8b267703b8bebab06e17e69a4a4de611592)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d819160
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d819160
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d819160

Branch: refs/heads/branch-2
Commit: 9d8191604010c15a2b9c1eec6e09e04821e3fcee
Parents: dcf1f1f
Author: Lei Xu 
Authored: Tue Jul 28 16:55:51 2015 -0700
Committer: Lei Xu 
Committed: Tue Jul 28 23:41:44 2015 -0700

--
 .../org/apache/hadoop/hdfs/TestReplication.java | 26 +++-
 1 file changed, 25 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d819160/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
index 010a26e..55e9056 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
@@ -18,6 +18,8 @@
 package org.apache.hadoop.hdfs;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
 import java.io.File;
@@ -25,6 +27,10 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.io.RandomAccessFile;
 import java.net.InetSocketAddress;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.Iterator;
 import java.util.Random;
@@ -514,12 +520,28 @@ public class TestReplication {
 if (data_dir.listFiles().length == 0) {
   nonParticipatedNodeDirs.add(data_dir);
 } else {
+  assertNull("participatedNodeDirs has already been set.",
+  participatedNodeDirs);
   participatedNodeDirs = data_dir;
 }
   }
+  assertEquals(2, nonParticipatedNodeDirs.size());
 
   String blockFile = null;
-  File[] listFiles = participatedNodeDirs.listFiles();
+  final List listFiles = new ArrayList<>();
+  Files.walkFileTree(participatedNodeDirs.toPath(),
+  new SimpleFileVisitor() {
+@Override
+public FileVisitResult visitFile(
+java.nio.file.Path file, BasicFileAttributes attrs)
+throws IOException {
+  listFiles.add(file.toFile());
+  return FileVisitResult.CONTINUE;
+}
+  }
+  );
+  assertFalse(listFiles.isEmpty());
+  int numReplicaCreated = 0;
   for (File file : listFiles) {
 if (file.getName().startsWith(Block.BLOCK_FILE_PREFIX)
 && !file.getName().endsWith("meta")) {
@@ -528,10 +550,12 @@ public class TestReplication {
 file1.mkdirs();
 new File(file1, blockFile).createNewFile();
 new File(file1, blockFile + "_1000.meta").createNewFile();
+numReplicaCreated++;
   }
   break;
 }
   }
+  assertEquals(2, numReplicaCreated);
 
   fs.setReplication(new Path("/test"), (short) 3);
   cluster.restartDataNodes(); // Lets detect all DNs about dummy copied



[2/2] hadoop git commit: HDFS-8822. Add SSD storagepolicy tests in TestBlockStoragePolicy#testDefaultPolicies (Contributed by Vinayakumar B)

2015-07-28 Thread vinayakumarb
HDFS-8822. Add SSD storagepolicy tests in 
TestBlockStoragePolicy#testDefaultPolicies (Contributed by Vinayakumar B)

(cherry picked from commit 975e138df316f59e8bb0642e138d4b1170fb8184)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dcf1f1fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dcf1f1fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dcf1f1fd

Branch: refs/heads/branch-2
Commit: dcf1f1fdf861fff699c97c74c46c82b541214030
Parents: 0c09597
Author: Vinayakumar B 
Authored: Wed Jul 29 11:50:48 2015 +0530
Committer: Vinayakumar B 
Committed: Wed Jul 29 11:51:18 2015 +0530

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 +
 .../hadoop/hdfs/TestBlockStoragePolicy.java | 88 
 2 files changed, 75 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcf1f1fd/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ce521a3..28c455e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -417,6 +417,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-8811. Move BlockStoragePolicy name's constants from
 HdfsServerConstants.java to HdfsConstants.java (vinayakumarb)
 
+HDFS-8822. Add SSD storagepolicy tests in TestBlockStoragePolicy#
+testDefaultPolicies (vinayakumarb)
+
   OPTIMIZATIONS
 
 HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcf1f1fd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 5875b13..ce3fce3 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -68,7 +68,11 @@ public class TestBlockStoragePolicy {
   static final EnumSet none = EnumSet.noneOf(StorageType.class);
   static final EnumSet archive = EnumSet.of(StorageType.ARCHIVE);
   static final EnumSet disk = EnumSet.of(StorageType.DISK);
-  static final EnumSet both = EnumSet.of(StorageType.DISK, 
StorageType.ARCHIVE);
+  static final EnumSet ssd = EnumSet.of(StorageType.SSD);
+  static final EnumSet disk_archive = EnumSet.of(StorageType.DISK,
+  StorageType.ARCHIVE);
+  static final EnumSet all = EnumSet.of(StorageType.SSD,
+  StorageType.DISK, StorageType.ARCHIVE);
 
   static final long FILE_LEN = 1024;
   static final short REPLICATION = 3;
@@ -153,8 +157,8 @@ public class TestBlockStoragePolicy {
 final List computed = 
cold.chooseStorageTypes(replication);
 assertStorageType(computed, replication, StorageType.ARCHIVE);
   }
-  assertCreationFallback(cold, null, null, null);
-  assertReplicationFallback(cold, null, null, null);
+  assertCreationFallback(cold, null, null, null, null, null);
+  assertReplicationFallback(cold, null, null, null, null);
 }
 
 { // check Warm policy
@@ -163,8 +167,10 @@ public class TestBlockStoragePolicy {
 final List computed = 
warm.chooseStorageTypes(replication);
 assertStorageType(computed, replication, StorageType.DISK, 
StorageType.ARCHIVE);
   }
-  assertCreationFallback(warm, StorageType.DISK, StorageType.DISK, 
StorageType.ARCHIVE);
-  assertReplicationFallback(warm, StorageType.DISK, StorageType.DISK, 
StorageType.ARCHIVE);
+  assertCreationFallback(warm, StorageType.DISK, StorageType.DISK,
+  StorageType.ARCHIVE, StorageType.DISK, null);
+  assertReplicationFallback(warm, StorageType.DISK, StorageType.DISK,
+  StorageType.ARCHIVE, StorageType.DISK);
 }
 
 { // check Hot policy
@@ -173,8 +179,50 @@ public class TestBlockStoragePolicy {
 final List computed = hot.chooseStorageTypes(replication);
 assertStorageType(computed, replication, StorageType.DISK);
   }
-  assertCreationFallback(hot, null, null, null);
-  assertReplicationFallback(hot, StorageType.ARCHIVE, null, 
StorageType.ARCHIVE);
+  assertCreationFallback(hot, null, null, null, null, null);
+  assertReplicationFallback(hot, StorageType.ARCHIVE, null,
+  StorageType.ARCHIVE, StorageType.ARCHIVE);
+}
+
+{ // check ONE_SSD policy
+  final BlockStoragePolicy onessd = POLICY_SUITE.getPolicy(ONESSD);

[1/2] hadoop git commit: HDFS-8822. Add SSD storagepolicy tests in TestBlockStoragePolicy#testDefaultPolicies (Contributed by Vinayakumar B)

2015-07-28 Thread vinayakumarb
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 0c0959794 -> dcf1f1fdf
  refs/heads/trunk 50887e5b0 -> 975e138df


HDFS-8822. Add SSD storagepolicy tests in 
TestBlockStoragePolicy#testDefaultPolicies (Contributed by Vinayakumar B)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/975e138d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/975e138d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/975e138d

Branch: refs/heads/trunk
Commit: 975e138df316f59e8bb0642e138d4b1170fb8184
Parents: 50887e5
Author: Vinayakumar B 
Authored: Wed Jul 29 11:50:48 2015 +0530
Committer: Vinayakumar B 
Committed: Wed Jul 29 11:50:48 2015 +0530

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 +
 .../hadoop/hdfs/TestBlockStoragePolicy.java | 88 
 2 files changed, 75 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/975e138d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 654c40f..d30c258 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -760,6 +760,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-8811. Move BlockStoragePolicy name's constants from
 HdfsServerConstants.java to HdfsConstants.java (vinayakumarb)
 
+HDFS-8822. Add SSD storagepolicy tests in TestBlockStoragePolicy#
+testDefaultPolicies (vinayakumarb)
+
   OPTIMIZATIONS
 
 HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/975e138d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 78e1e58..631d9f7 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -67,7 +67,11 @@ public class TestBlockStoragePolicy {
   static final EnumSet none = EnumSet.noneOf(StorageType.class);
   static final EnumSet archive = EnumSet.of(StorageType.ARCHIVE);
   static final EnumSet disk = EnumSet.of(StorageType.DISK);
-  static final EnumSet both = EnumSet.of(StorageType.DISK, 
StorageType.ARCHIVE);
+  static final EnumSet ssd = EnumSet.of(StorageType.SSD);
+  static final EnumSet disk_archive = EnumSet.of(StorageType.DISK,
+  StorageType.ARCHIVE);
+  static final EnumSet all = EnumSet.of(StorageType.SSD,
+  StorageType.DISK, StorageType.ARCHIVE);
 
   static final long FILE_LEN = 1024;
   static final short REPLICATION = 3;
@@ -153,8 +157,8 @@ public class TestBlockStoragePolicy {
 final List computed = 
cold.chooseStorageTypes(replication);
 assertStorageType(computed, replication, StorageType.ARCHIVE);
   }
-  assertCreationFallback(cold, null, null, null);
-  assertReplicationFallback(cold, null, null, null);
+  assertCreationFallback(cold, null, null, null, null, null);
+  assertReplicationFallback(cold, null, null, null, null);
 }
 
 { // check Warm policy
@@ -163,8 +167,10 @@ public class TestBlockStoragePolicy {
 final List computed = 
warm.chooseStorageTypes(replication);
 assertStorageType(computed, replication, StorageType.DISK, 
StorageType.ARCHIVE);
   }
-  assertCreationFallback(warm, StorageType.DISK, StorageType.DISK, 
StorageType.ARCHIVE);
-  assertReplicationFallback(warm, StorageType.DISK, StorageType.DISK, 
StorageType.ARCHIVE);
+  assertCreationFallback(warm, StorageType.DISK, StorageType.DISK,
+  StorageType.ARCHIVE, StorageType.DISK, null);
+  assertReplicationFallback(warm, StorageType.DISK, StorageType.DISK,
+  StorageType.ARCHIVE, StorageType.DISK);
 }
 
 { // check Hot policy
@@ -173,8 +179,50 @@ public class TestBlockStoragePolicy {
 final List computed = hot.chooseStorageTypes(replication);
 assertStorageType(computed, replication, StorageType.DISK);
   }
-  assertCreationFallback(hot, null, null, null);
-  assertReplicationFallback(hot, StorageType.ARCHIVE, null, 
StorageType.ARCHIVE);
+  assertCreationFallback(hot, null, null, null, null, null);
+  assertReplicationFallback(hot, StorageType.ARCHIVE, null,
+  StorageType.ARCHIVE, StorageType.ARCHIVE);
+}
+
+{ // check ONE_SSD policy
+  final BlockSt

[2/2] hadoop git commit: HDFS-8811. Move BlockStoragePolicy name's constants from HdfsServerConstants.java to HdfsConstants.java (Contributed by Vinayakumar B)

2015-07-28 Thread vinayakumarb
HDFS-8811. Move BlockStoragePolicy name's constants from 
HdfsServerConstants.java to HdfsConstants.java (Contributed by Vinayakumar B)

(cherry picked from commit 50887e5b07b6abb20c0edd74211e5612dc7b16da)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c095979
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c095979
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c095979

Branch: refs/heads/branch-2
Commit: 0c09597942284d99db729d9a9eb4df682886078b
Parents: 71a5aa2
Author: Vinayakumar B 
Authored: Wed Jul 29 11:45:33 2015 +0530
Committer: Vinayakumar B 
Committed: Wed Jul 29 11:46:19 2015 +0530

--
 .../hadoop/hdfs/protocol/HdfsConstants.java |  3 ++
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../BlockStoragePolicySuite.java|  6 ++--
 .../hdfs/server/common/HdfsServerConstants.java |  3 --
 .../org/apache/hadoop/hdfs/DFSTestUtil.java |  2 +-
 .../hadoop/hdfs/TestBlockStoragePolicy.java | 30 ++--
 .../hdfs/server/mover/TestStorageMover.java |  8 +++---
 7 files changed, 29 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c095979/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index f12a6fb..d4fb860 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -37,6 +37,9 @@ public class HdfsConstants {
   public static final String MEMORY_STORAGE_POLICY_NAME = "LAZY_PERSIST";
   public static final String ALLSSD_STORAGE_POLICY_NAME = "ALL_SSD";
   public static final String ONESSD_STORAGE_POLICY_NAME = "ONE_SSD";
+  public static final String HOT_STORAGE_POLICY_NAME = "HOT";
+  public static final String WARM_STORAGE_POLICY_NAME = "WARM";
+  public static final String COLD_STORAGE_POLICY_NAME = "COLD";
   // TODO should be conf injected?
   public static final int DEFAULT_DATA_SOCKET_SIZE = 128 * 1024;
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c095979/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1efea0f..ce521a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -414,6 +414,9 @@ Release 2.8.0 - UNRELEASED
 
 HDFS-8180. AbstractFileSystem Implementation for WebHdfs. (snayak via 
jghoman)
 
+HDFS-8811. Move BlockStoragePolicy name's constants from
+HdfsServerConstants.java to HdfsConstants.java (vinayakumarb)
+
   OPTIMIZATIONS
 
 HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c095979/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
index 2a71c29..7e6c2b2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
@@ -69,18 +69,18 @@ public class BlockStoragePolicySuite {
 new StorageType[]{StorageType.SSD, StorageType.DISK});
 final byte hotId = HdfsServerConstants.HOT_STORAGE_POLICY_ID;
 policies[hotId] = new BlockStoragePolicy(hotId,
-HdfsServerConstants.HOT_STORAGE_POLICY_NAME,
+HdfsConstants.HOT_STORAGE_POLICY_NAME,
 new StorageType[]{StorageType.DISK}, StorageType.EMPTY_ARRAY,
 new StorageType[]{StorageType.ARCHIVE});
 final byte warmId = HdfsServerConstants.WARM_STORAGE_POLICY_ID;
 policies[warmId] = new BlockStoragePolicy(warmId,
-HdfsServerConstants.WARM_STORAGE_POLICY_NAME,
+HdfsConstants.WARM_STORAGE_POLICY_NAME,
 new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
 new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
 new 

[1/2] hadoop git commit: HDFS-8811. Move BlockStoragePolicy name's constants from HdfsServerConstants.java to HdfsConstants.java (Contributed by Vinayakumar B)

2015-07-28 Thread vinayakumarb
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 71a5aa267 -> 0c0959794
  refs/heads/trunk 0712a8103 -> 50887e5b0


HDFS-8811. Move BlockStoragePolicy name's constants from 
HdfsServerConstants.java to HdfsConstants.java (Contributed by Vinayakumar B)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50887e5b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50887e5b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50887e5b

Branch: refs/heads/trunk
Commit: 50887e5b07b6abb20c0edd74211e5612dc7b16da
Parents: 0712a81
Author: Vinayakumar B 
Authored: Wed Jul 29 11:45:33 2015 +0530
Committer: Vinayakumar B 
Committed: Wed Jul 29 11:45:33 2015 +0530

--
 .../hadoop/hdfs/protocol/HdfsConstants.java |  3 ++
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../BlockStoragePolicySuite.java|  6 ++--
 .../hdfs/server/common/HdfsServerConstants.java |  3 --
 .../org/apache/hadoop/hdfs/DFSTestUtil.java |  2 +-
 .../hadoop/hdfs/TestBlockStoragePolicy.java | 30 ++--
 .../hdfs/server/mover/TestStorageMover.java |  8 +++---
 7 files changed, 29 insertions(+), 26 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/50887e5b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 58c7ea1..443576d 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -37,6 +37,9 @@ public final class HdfsConstants {
   public static final String MEMORY_STORAGE_POLICY_NAME = "LAZY_PERSIST";
   public static final String ALLSSD_STORAGE_POLICY_NAME = "ALL_SSD";
   public static final String ONESSD_STORAGE_POLICY_NAME = "ONE_SSD";
+  public static final String HOT_STORAGE_POLICY_NAME = "HOT";
+  public static final String WARM_STORAGE_POLICY_NAME = "WARM";
+  public static final String COLD_STORAGE_POLICY_NAME = "COLD";
   // TODO should be conf injected?
   public static final int DEFAULT_DATA_SOCKET_SIZE = 128 * 1024;
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/50887e5b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ef12720..654c40f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -757,6 +757,9 @@ Release 2.8.0 - UNRELEASED
 
 HDFS-8180. AbstractFileSystem Implementation for WebHdfs. (snayak via 
jghoman)
 
+HDFS-8811. Move BlockStoragePolicy name's constants from
+HdfsServerConstants.java to HdfsConstants.java (vinayakumarb)
+
   OPTIMIZATIONS
 
 HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/50887e5b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
index 2a71c29..7e6c2b2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
@@ -69,18 +69,18 @@ public class BlockStoragePolicySuite {
 new StorageType[]{StorageType.SSD, StorageType.DISK});
 final byte hotId = HdfsServerConstants.HOT_STORAGE_POLICY_ID;
 policies[hotId] = new BlockStoragePolicy(hotId,
-HdfsServerConstants.HOT_STORAGE_POLICY_NAME,
+HdfsConstants.HOT_STORAGE_POLICY_NAME,
 new StorageType[]{StorageType.DISK}, StorageType.EMPTY_ARRAY,
 new StorageType[]{StorageType.ARCHIVE});
 final byte warmId = HdfsServerConstants.WARM_STORAGE_POLICY_ID;
 policies[warmId] = new BlockStoragePolicy(warmId,
-HdfsServerConstants.WARM_STORAGE_POLICY_NAME,
+HdfsConstants.WARM_STORAGE_POLICY_NAME,
 new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
 new StorageT

hadoop git commit: HDFS-8180. AbstractFileSystem Implementation for WebHdfs. Contributed by Sathosh G Nayak.

2015-07-28 Thread jghoman
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 587593566 -> 71a5aa267


HDFS-8180. AbstractFileSystem Implementation for WebHdfs. Contributed by 
Sathosh G Nayak.

(cherry picked from commit 0712a8103fec6e9a9ceba335e3c3800b85b2c7ca)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71a5aa26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71a5aa26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71a5aa26

Branch: refs/heads/branch-2
Commit: 71a5aa267d8e93308f2546002973cefc89c8a8f8
Parents: 5875935
Author: Jakob Homan 
Authored: Tue Jul 28 21:03:31 2015 -0700
Committer: Jakob Homan 
Committed: Tue Jul 28 21:11:02 2015 -0700

--
 .../src/main/resources/core-default.xml |  12 ++
 .../fs/FileContextMainOperationsBaseTest.java   |   4 +-
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |   2 +
 .../java/org/apache/hadoop/fs/SWebHdfs.java |  51 ++
 .../main/java/org/apache/hadoop/fs/WebHdfs.java |  51 ++
 .../main/java/org/apache/hadoop/fs/package.html |  26 +++
 .../TestSWebHdfsFileContextMainOperations.java  | 110 +
 .../TestWebHdfsFileContextMainOperations.java   | 157 +++
 8 files changed, 411 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a5aa26/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml 
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 8643a0e..ee42675 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -597,6 +597,18 @@ for ldap providers in the same way as above does.
 
 
 
+  fs.AbstractFileSystem.webhdfs.impl
+  org.apache.hadoop.fs.WebHdfs
+  The FileSystem for webhdfs: uris.
+
+
+
+  fs.AbstractFileSystem.swebhdfs.impl
+  org.apache.hadoop.fs.SWebHdfs
+  The FileSystem for swebhdfs: uris.
+
+
+
   fs.ftp.host
   0.0.0.0
   FTP filesystem connects to this server

http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a5aa26/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
index e872176..12ec375 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
@@ -1249,7 +1249,7 @@ public abstract class FileContextMainOperationsBaseTest  {
 byte[] bb = new byte[(int)len];
 FSDataInputStream fsdis = fc.open(path);
 try {
-  fsdis.read(bb);
+  fsdis.readFully(bb);
 } finally {
   fsdis.close();
 }
@@ -1310,7 +1310,7 @@ public abstract class FileContextMainOperationsBaseTest  {
 byte[] bb = new byte[data.length];
 FSDataInputStream fsdis = fc.open(path);
 try {
-  fsdis.read(bb);
+  fsdis.readFully(bb);
 } finally {
   fsdis.close();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a5aa26/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9dc2a82..1efea0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -412,6 +412,8 @@ Release 2.8.0 - UNRELEASED
 
 HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)
 
+HDFS-8180. AbstractFileSystem Implementation for WebHdfs. (snayak via 
jghoman)
+
   OPTIMIZATIONS
 
 HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a5aa26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
new file mode 100644
index 000..cd36393
--- /dev/null
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more con

hadoop git commit: HDFS-8180. AbstractFileSystem Implementation for WebHdfs. Contributed by Sathosh G Nayak.

2015-07-28 Thread jghoman
Repository: hadoop
Updated Branches:
  refs/heads/trunk 69b095730 -> 0712a8103


HDFS-8180. AbstractFileSystem Implementation for WebHdfs. Contributed by 
Sathosh G Nayak.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0712a810
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0712a810
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0712a810

Branch: refs/heads/trunk
Commit: 0712a8103fec6e9a9ceba335e3c3800b85b2c7ca
Parents: 69b0957
Author: Jakob Homan 
Authored: Tue Jul 28 21:03:31 2015 -0700
Committer: Jakob Homan 
Committed: Tue Jul 28 21:03:31 2015 -0700

--
 .../src/main/resources/core-default.xml |  12 ++
 .../fs/FileContextMainOperationsBaseTest.java   |   4 +-
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |   2 +
 .../java/org/apache/hadoop/fs/SWebHdfs.java |  51 ++
 .../main/java/org/apache/hadoop/fs/WebHdfs.java |  51 ++
 .../main/java/org/apache/hadoop/fs/package.html |  26 +++
 .../TestSWebHdfsFileContextMainOperations.java  | 110 +
 .../TestWebHdfsFileContextMainOperations.java   | 157 +++
 8 files changed, 411 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0712a810/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml 
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 13702ee..bfdd453 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -596,6 +596,18 @@ for ldap providers in the same way as above does.
 
 
 
+  fs.AbstractFileSystem.webhdfs.impl
+  org.apache.hadoop.fs.WebHdfs
+  The FileSystem for webhdfs: uris.
+
+
+
+  fs.AbstractFileSystem.swebhdfs.impl
+  org.apache.hadoop.fs.SWebHdfs
+  The FileSystem for swebhdfs: uris.
+
+
+
   fs.ftp.host
   0.0.0.0
   FTP filesystem connects to this server

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0712a810/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
index e872176..12ec375 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
@@ -1249,7 +1249,7 @@ public abstract class FileContextMainOperationsBaseTest  {
 byte[] bb = new byte[(int)len];
 FSDataInputStream fsdis = fc.open(path);
 try {
-  fsdis.read(bb);
+  fsdis.readFully(bb);
 } finally {
   fsdis.close();
 }
@@ -1310,7 +1310,7 @@ public abstract class FileContextMainOperationsBaseTest  {
 byte[] bb = new byte[data.length];
 FSDataInputStream fsdis = fc.open(path);
 try {
-  fsdis.read(bb);
+  fsdis.readFully(bb);
 } finally {
   fsdis.close();
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0712a810/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9b2de81..ef12720 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -755,6 +755,8 @@ Release 2.8.0 - UNRELEASED
 
 HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)
 
+HDFS-8180. AbstractFileSystem Implementation for WebHdfs. (snayak via 
jghoman)
+
   OPTIMIZATIONS
 
 HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0712a810/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
new file mode 100644
index 000..cd36393
--- /dev/null
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this 

[07/50] [abbrv] hadoop git commit: YARN-3941. Proportional Preemption policy should try to avoid sending duplicate PREEMPT_CONTAINER event to scheduler. (Sunil G via wangda)

2015-07-28 Thread jianhe
YARN-3941. Proportional Preemption policy should try to avoid sending duplicate 
PREEMPT_CONTAINER event to scheduler. (Sunil G via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3bba1800
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3bba1800
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3bba1800

Branch: refs/heads/YARN-1197
Commit: 3bba1800513b38a4827f7552f348db87dc47c783
Parents: ee98d63
Author: Wangda Tan 
Authored: Thu Jul 23 10:07:57 2015 -0700
Committer: Wangda Tan 
Committed: Thu Jul 23 10:07:57 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt | 2 ++
 .../capacity/ProportionalCapacityPreemptionPolicy.java  | 9 ++---
 .../capacity/TestProportionalCapacityPreemptionPolicy.java  | 6 +++---
 3 files changed, 11 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bba1800/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 93962f1..9416cd6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -664,6 +664,8 @@ Release 2.8.0 - UNRELEASED
 
 YARN-3956. Fix TestNodeManagerHardwareUtils fails on Mac (Varun Vasudev 
via wangda)
 
+YARN-3941. Proportional Preemption policy should try to avoid sending 
duplicate PREEMPT_CONTAINER event to scheduler. (Sunil G via wangda)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bba1800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 1152cef..77df059 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -260,13 +260,16 @@ public class ProportionalCapacityPreemptionPolicy 
implements SchedulingEditPolic
   SchedulerEventType.KILL_CONTAINER));
   preempted.remove(container);
 } else {
+  if (preempted.get(container) != null) {
+// We already updated the information to scheduler earlier, we need
+// not have to raise another event.
+continue;
+  }
   //otherwise just send preemption events
   rmContext.getDispatcher().getEventHandler().handle(
   new ContainerPreemptEvent(appAttemptId, container,
   SchedulerEventType.PREEMPT_CONTAINER));
-  if (preempted.get(container) == null) {
-preempted.put(container, clock.getTime());
-  }
+  preempted.put(container, clock.getTime());
 }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bba1800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index bc4d0dc..8d9f48a 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/mo

[03/50] [abbrv] hadoop git commit: YARN-3954. Fix TestYarnConfigurationFields#testCompareConfigurationClassAgainstXml. (varun saxena via rohithsharmaks)

2015-07-28 Thread jianhe
YARN-3954. Fix 
TestYarnConfigurationFields#testCompareConfigurationClassAgainstXml. (varun 
saxena via rohithsharmaks)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8376ea32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8376ea32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8376ea32

Branch: refs/heads/YARN-1197
Commit: 8376ea3297a3eab33df27454b18cf215cfb7c6ff
Parents: 76ec26d
Author: rohithsharmaks 
Authored: Thu Jul 23 00:28:24 2015 +0530
Committer: rohithsharmaks 
Committed: Thu Jul 23 00:28:24 2015 +0530

--
 hadoop-yarn-project/CHANGES.txt   |  3 +++
 .../src/main/resources/yarn-default.xml   | 10 ++
 2 files changed, 13 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/8376ea32/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f751862..eb52745 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -656,6 +656,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo 
 should based on total-used-resources. (Bibin A Chundatt via wangda)
 
+YARN-3954. Fix 
TestYarnConfigurationFields#testCompareConfigurationClassAgainstXml.
+(varun saxena via rohithsharmaks)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/8376ea32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 2edeef0..d586f51 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2131,4 +2131,14 @@
 false
   
 
+  
+
+Defines maximum application priority in a cluster.
+If an application is submitted with a priority higher than this value, it 
will be
+reset to this maximum value.
+
+yarn.cluster.max-application-priority
+0
+  
+
 



[40/50] [abbrv] hadoop git commit: YARN-3853. Add docker container runtime support to LinuxContainterExecutor. Contributed by Sidharta Seethana.

2015-07-28 Thread jianhe
YARN-3853. Add docker container runtime support to LinuxContainterExecutor. 
Contributed by Sidharta Seethana.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e6fce91
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e6fce91
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e6fce91

Branch: refs/heads/YARN-1197
Commit: 3e6fce91a471b4a5099de109582e7c6417e8a822
Parents: f36835f
Author: Varun Vasudev 
Authored: Mon Jul 27 11:57:40 2015 -0700
Committer: Varun Vasudev 
Committed: Mon Jul 27 11:57:40 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |   4 +
 .../server/nodemanager/ContainerExecutor.java   |  23 +-
 .../nodemanager/DefaultContainerExecutor.java   |   2 +-
 .../nodemanager/DockerContainerExecutor.java|   2 +-
 .../nodemanager/LinuxContainerExecutor.java | 222 +++
 .../launcher/ContainerLaunch.java   |  15 +
 .../linux/privileged/PrivilegedOperation.java   |  46 +++-
 .../PrivilegedOperationException.java   |  30 +-
 .../privileged/PrivilegedOperationExecutor.java |  30 +-
 .../linux/resources/CGroupsHandler.java |   8 +
 .../linux/resources/CGroupsHandlerImpl.java |  12 +-
 .../runtime/DefaultLinuxContainerRuntime.java   | 148 ++
 .../DelegatingLinuxContainerRuntime.java| 110 
 .../runtime/DockerLinuxContainerRuntime.java| 273 +++
 .../linux/runtime/LinuxContainerRuntime.java|  38 +++
 .../runtime/LinuxContainerRuntimeConstants.java |  69 +
 .../linux/runtime/docker/DockerClient.java  |  82 ++
 .../linux/runtime/docker/DockerCommand.java |  66 +
 .../linux/runtime/docker/DockerLoadCommand.java |  30 ++
 .../linux/runtime/docker/DockerRunCommand.java  | 107 
 .../runtime/ContainerExecutionException.java|  85 ++
 .../runtime/ContainerRuntime.java   |  50 
 .../runtime/ContainerRuntimeConstants.java  |  33 +++
 .../runtime/ContainerRuntimeContext.java| 105 +++
 .../executor/ContainerLivenessContext.java  |  13 +
 .../executor/ContainerReacquisitionContext.java |  13 +
 .../executor/ContainerSignalContext.java|  13 +
 .../executor/ContainerStartContext.java |  23 +-
 .../TestLinuxContainerExecutorWithMocks.java| 118 +---
 .../TestPrivilegedOperationExecutor.java|   8 +-
 .../runtime/TestDockerContainerRuntime.java | 219 +++
 31 files changed, 1815 insertions(+), 182 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4e54aea..534c55a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -153,6 +153,10 @@ Release 2.8.0 - UNRELEASED
 YARN-3852. Add docker container support to container-executor
 (Abin Shahab via vvasudev)
 
+YARN-3853. Add docker container runtime support to LinuxContainterExecutor.
+(Sidharta Seethana via vvasudev)
+
+
   IMPROVEMENTS
 
 YARN-644. Basic null check is not performed on passed in arguments before

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 79f9b0d..68bfbbf 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -24,8 +24,10 @@ import java.io.OutputStream;
 import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -39,6 +41,7 @@ import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.apac

[27/50] [abbrv] hadoop git commit: HADOOP-11807. add a lint mode to releasedocmaker (ramtin via aw)

2015-07-28 Thread jianhe
HADOOP-11807. add a lint mode to releasedocmaker (ramtin via aw)

(cherry picked from commit 8e657fba2fd33f7550597ea9c4c6e9a87aa1ef1c)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/098ba450
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/098ba450
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/098ba450

Branch: refs/heads/YARN-1197
Commit: 098ba450cc98475b84d60bb5ac3bd7b558b2a67c
Parents: a3bd7b4
Author: Allen Wittenauer 
Authored: Sat Jun 27 08:59:50 2015 -0700
Committer: Andrew Wang 
Committed: Fri Jul 24 18:31:23 2015 -0700

--
 dev-support/releasedocmaker.py  | 76 +---
 hadoop-common-project/hadoop-common/CHANGES.txt |  2 +
 2 files changed, 68 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/098ba450/dev-support/releasedocmaker.py
--
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index 2ccc1c0..8e68b3c 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -87,8 +87,15 @@ def notableclean(str):
   str=str.rstrip()
   return str
 
+# clean output dir
+def cleanOutputDir(dir):
+files = os.listdir(dir)
+for name in files:
+os.remove(os.path.join(dir,name))
+os.rmdir(dir)
+
 def mstr(obj):
-  if (obj == None):
+  if (obj is None):
 return ""
   return unicode(obj)
 
@@ -148,7 +155,7 @@ class Jira:
 return mstr(self.fields['description'])
 
   def getReleaseNote(self):
-if (self.notes == None):
+if (self.notes is None):
   field = self.parent.fieldIdMap['Release Note']
   if (self.fields.has_key(field)):
 self.notes=mstr(self.fields[field])
@@ -159,14 +166,14 @@ class Jira:
   def getPriority(self):
 ret = ""
 pri = self.fields['priority']
-if(pri != None):
+if(pri is not None):
   ret = pri['name']
 return mstr(ret)
 
   def getAssignee(self):
 ret = ""
 mid = self.fields['assignee']
-if(mid != None):
+if(mid is not None):
   ret = mid['displayName']
 return mstr(ret)
 
@@ -182,21 +189,21 @@ class Jira:
   def getType(self):
 ret = ""
 mid = self.fields['issuetype']
-if(mid != None):
+if(mid is not None):
   ret = mid['name']
 return mstr(ret)
 
   def getReporter(self):
 ret = ""
 mid = self.fields['reporter']
-if(mid != None):
+if(mid is not None):
   ret = mid['displayName']
 return mstr(ret)
 
   def getProject(self):
 ret = ""
 mid = self.fields['project']
-if(mid != None):
+if(mid is not None):
   ret = mid['key']
 return mstr(ret)
 
@@ -214,7 +221,7 @@ class Jira:
 return False
 
   def getIncompatibleChange(self):
-if (self.incompat == None):
+if (self.incompat is None):
   field = self.parent.fieldIdMap['Hadoop Flags']
   self.reviewed=False
   self.incompat=False
@@ -227,6 +234,24 @@ class Jira:
   self.reviewed=True
 return self.incompat
 
+  def checkMissingComponent(self):
+  if (len(self.fields['components'])>0):
+  return False
+  return True
+
+  def checkMissingAssignee(self):
+  if (self.fields['assignee'] is not None):
+  return False
+  return True
+
+  def checkVersionString(self):
+  field = self.parent.fieldIdMap['Fix Version/s']
+  for h in self.fields[field]:
+  found = re.match('^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', h['name'])
+  if not found:
+  return True
+  return False
+
   def getReleaseDate(self,version):
 for j in range(len(self.fields['fixVersions'])):
   if self.fields['fixVersions'][j]==version:
@@ -339,9 +364,11 @@ def main():
  help="build an index file")
   parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
  help="use current date for unreleased versions")
+  parser.add_option("-n","--lint", dest="lint", action="store_true",
+ help="use lint flag to exit on failures")
   (options, args) = parser.parse_args()
 
-  if (options.versions == None):
+  if (options.versions is None):
 options.versions = []
 
   if (len(args) > 2):
@@ -396,6 +423,9 @@ def main():
   reloutputs.writeAll(relhead)
   choutputs.writeAll(chhead)
 
+  errorCount=0
+  warningCount=0
+  lintMessage=""
   incompatlist=[]
   buglist=[]
   improvementlist=[]
@@ -408,6 +438,14 @@ def main():
   for jira in sorted(jlist):
 if jira.getIncompatibleChange():
   incompatlist.append(jira)
+  if (len(jira.getReleaseNote())==0):
+  warningCount+=1
+
+if jira.checkVersionString():
+   warningCount+=1
+
+if jira.checkMissingComponent() or jira.checkMissingAssignee():
+  errorCount+=1
 elif jira.getType() =

[47/50] [abbrv] hadoop git commit: YARN-3866. AM-RM protocol changes to support container resizing. Contributed by Meng Ding

2015-07-28 Thread jianhe
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7758afd3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java
deleted file mode 100644
index 29b0ffe..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease;
-import org.apache.hadoop.yarn.api.records.Resource;
-import 
org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceDecreasePBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProto;
-import org.junit.Test;
-
-public class TestContainerResourceDecrease {
-  @Test
-  public void testResourceDecreaseContext() {
-ContainerId containerId = ContainerId
-.newContainerId(ApplicationAttemptId.newInstance(
-ApplicationId.newInstance(1234, 3), 3), 7);
-Resource resource = Resource.newInstance(1023, 3);
-ContainerResourceDecrease ctx = ContainerResourceDecrease.newInstance(
-containerId, resource);
-
-// get proto and recover to ctx
-ContainerResourceDecreaseProto proto = 
-((ContainerResourceDecreasePBImpl) ctx).getProto();
-ctx = new ContainerResourceDecreasePBImpl(proto);
-
-// check values
-Assert.assertEquals(ctx.getCapability(), resource);
-Assert.assertEquals(ctx.getContainerId(), containerId);
-  }
-  
-  @Test
-  public void testResourceDecreaseContextWithNull() {
-ContainerResourceDecrease ctx = ContainerResourceDecrease.newInstance(null,
-null);
-
-// get proto and recover to ctx;
-ContainerResourceDecreaseProto proto = 
-((ContainerResourceDecreasePBImpl) ctx).getProto();
-ctx = new ContainerResourceDecreasePBImpl(proto);
-
-// check values
-Assert.assertNull(ctx.getCapability());
-Assert.assertNull(ctx.getContainerId());
-  }
-}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7758afd3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java
deleted file mode 100644
index 932d5a7..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api;

[01/50] [abbrv] hadoop git commit: HADOOP-12184. Remove unused Linux-specific constants in NativeIO (Martin Walsh via Colin P. McCabe)

2015-07-28 Thread jianhe
Repository: hadoop
Updated Branches:
  refs/heads/YARN-1197 2934ff345 -> cb9566269 (forced update)


HADOOP-12184. Remove unused Linux-specific constants in NativeIO (Martin Walsh 
via Colin P. McCabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b3bceb5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b3bceb5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b3bceb5

Branch: refs/heads/YARN-1197
Commit: 1b3bceb58c8e536a75fa3f99cc3ceeaba91a07de
Parents: efa9724
Author: Colin Patrick Mccabe 
Authored: Wed Jul 22 11:11:38 2015 -0700
Committer: Colin Patrick Mccabe 
Committed: Wed Jul 22 11:34:10 2015 -0700

--
 hadoop-common-project/hadoop-common/CHANGES.txt  | 3 +++
 .../src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java| 4 
 2 files changed, 3 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b3bceb5/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index c0e5c92..ff7d2ad 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -713,6 +713,9 @@ Release 2.8.0 - UNRELEASED
 HADOOP-12214. Parse 'HadoopArchive' commandline using cli Options.
 (vinayakumarb)
 
+HADOOP-12184. Remove unused Linux-specific constants in NativeIO (Martin
+Walsh via Colin P. McCabe)
+
   OPTIMIZATIONS
 
 HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b3bceb5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
index 688b955..77a40ea 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
@@ -67,9 +67,6 @@ public class NativeIO {
 public static final int O_APPEND   = 02000;
 public static final int O_NONBLOCK = 04000;
 public static final int O_SYNC   =  01;
-public static final int O_ASYNC  =  02;
-public static final int O_FSYNC = O_SYNC;
-public static final int O_NDELAY = O_NONBLOCK;
 
 // Flags for posix_fadvise() from bits/fcntl.h
 /* No further special treatment.  */
@@ -356,7 +353,6 @@ public class NativeIO {
   public static final int   S_IFREG  = 010;  /* regular */
   public static final int   S_IFLNK  = 012;  /* symbolic link */
   public static final int   S_IFSOCK = 014;  /* socket */
-  public static final int   S_IFWHT  = 016;  /* whiteout */
   public static final int S_ISUID = 0004000;  /* set user id on execution 
*/
   public static final int S_ISGID = 0002000;  /* set group id on execution 
*/
   public static final int S_ISVTX = 0001000;  /* save swapped text even 
after use */



[15/50] [abbrv] hadoop git commit: YARN-3845. Scheduler page does not render RGBA color combinations in IE11. (Contributed by Mohammad Shahid Khan)

2015-07-28 Thread jianhe
YARN-3845. Scheduler page does not render RGBA color combinations in IE11. 
(Contributed by Mohammad Shahid Khan)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e202efaf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e202efaf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e202efaf

Branch: refs/heads/YARN-1197
Commit: e202efaf932c940e6da5fe857ae55c0808fd4fdd
Parents: 02c0181
Author: Rohith Sharma K S 
Authored: Fri Jul 24 12:43:06 2015 +0530
Committer: Rohith Sharma K S 
Committed: Fri Jul 24 12:43:06 2015 +0530

--
 hadoop-yarn-project/CHANGES.txt   |  3 +++
 .../apache/hadoop/yarn/webapp/view/TwoColumnLayout.java   |  2 +-
 .../resourcemanager/webapp/CapacitySchedulerPage.java |  7 ---
 .../resourcemanager/webapp/DefaultSchedulerPage.java  |  4 ++--
 .../server/resourcemanager/webapp/FairSchedulerPage.java  | 10 ++
 5 files changed, 16 insertions(+), 10 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e202efaf/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3d41ba7..f23853b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -669,6 +669,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3900. Protobuf layout of yarn_security_token causes errors in other 
protos
 that include it (adhoot via rkanter)
 
+YARN-3845. Scheduler page does not render RGBA color combinations in IE11. 
+(Contributed by Mohammad Shahid Khan)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e202efaf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java
index b8f5f75..4d7752d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java
@@ -126,7 +126,7 @@ public class TwoColumnLayout extends HtmlPage {
 styles.add(join('#', tableId, "_paginate span {font-weight:normal}"));
 styles.add(join('#', tableId, " .progress {width:8em}"));
 styles.add(join('#', tableId, "_processing {top:-1.5em; font-size:1em;"));
-styles.add("  color:#000; background:rgba(255, 255, 255, 0.8)}");
+styles.add("  color:#000; background:#fefefe}");
 for (String style : innerStyles) {
   styles.add(join('#', tableId, " ", style));
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e202efaf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index a784601..12a3013 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -59,9 +59,10 @@ class CapacitySchedulerPage extends RmView {
   static final float Q_MAX_WIDTH = 0.8f;
   static final float Q_STATS_POS = Q_MAX_WIDTH + 0.05f;
   static final String Q_END = "left:101%";
-  static final String Q_GIVEN = "left:0%;background:none;border:1px dashed 
rgba(0,0,0,0.25)";
-  static final String Q_OVER = "background:rgba(255, 140, 0, 0.8)";
-  static final String Q_UNDER = "background:rgba(50, 205, 50, 0.8)";
+  static final String Q_GIVEN =
+  "left:0%;background:none;border:1px dashed #BFBFBF";
+  static final String Q_OVER = "background:#FFA333";
+  static final String Q_UNDER = "background:#5BD75B";
 
   @RequestScoped
   static class CSQInfo {

http://git-wip-us.apache.org/repos/asf

[09/50] [abbrv] hadoop git commit: HDFS-8730. Clean up the import statements in ClientProtocol. Contributed by Takanobu Asanuma.

2015-07-28 Thread jianhe
HDFS-8730. Clean up the import statements in ClientProtocol. Contributed by 
Takanobu Asanuma.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/813cf89b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/813cf89b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/813cf89b

Branch: refs/heads/YARN-1197
Commit: 813cf89bb56ad1a48b35fd44644d63540e8fa7d1
Parents: adfa34f
Author: Haohui Mai 
Authored: Thu Jul 23 10:30:17 2015 -0700
Committer: Haohui Mai 
Committed: Thu Jul 23 10:31:11 2015 -0700

--
 .../hadoop/hdfs/protocol/ClientProtocol.java| 306 +++
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |   3 +
 2 files changed, 182 insertions(+), 127 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/813cf89b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 381be30..713c23c 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -17,7 +17,6 @@
  */
 package org.apache.hadoop.hdfs.protocol;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.util.EnumSet;
 import java.util.List;
@@ -29,14 +28,9 @@ import 
org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
 import org.apache.hadoop.fs.CacheFlag;
 import org.apache.hadoop.fs.ContentSummary;
 import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
 import org.apache.hadoop.fs.FsServerDefaults;
-import org.apache.hadoop.fs.InvalidPathException;
 import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.Options.Rename;
-import org.apache.hadoop.fs.ParentNotDirectoryException;
 import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.fs.UnresolvedLinkException;
 import org.apache.hadoop.fs.XAttr;
 import org.apache.hadoop.fs.XAttrSetFlag;
 import org.apache.hadoop.fs.permission.AclEntry;
@@ -48,14 +42,11 @@ import 
org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
 import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
 import 
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
 import 
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
-import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
-import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
 import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
 import org.apache.hadoop.io.EnumSetWritable;
 import org.apache.hadoop.io.Text;
 import org.apache.hadoop.io.retry.AtMostOnce;
 import org.apache.hadoop.io.retry.Idempotent;
-import org.apache.hadoop.security.AccessControlException;
 import org.apache.hadoop.security.KerberosInfo;
 import org.apache.hadoop.security.token.Token;
 import org.apache.hadoop.security.token.TokenInfo;
@@ -121,9 +112,12 @@ public interface ClientProtocol {
*
* @return file length and array of blocks with their locations
*
-   * @throws AccessControlException If access is denied
-   * @throws FileNotFoundException If file src does not exist
-   * @throws UnresolvedLinkException If src contains a symlink
+   * @throws org.apache.hadoop.security.AccessControlException If access is
+   *   denied
+   * @throws java.io.FileNotFoundException If file src does not
+   *   exist
+   * @throws org.apache.hadoop.fs.UnresolvedLinkException If src
+   *   contains a symlink
* @throws IOException If an I/O error occurred
*/
   @Idempotent
@@ -166,24 +160,29 @@ public interface ClientProtocol {
*
* @return the status of the created file, it could be null if the server
*   doesn't support returning the file status
-   * @throws AccessControlException If access is denied
+   * @throws org.apache.hadoop.security.AccessControlException If access is
+   *   denied
* @throws AlreadyBeingCreatedException if the path does not exist.
* @throws DSQuotaExceededException If file creation violates disk space
*   quota restriction
-   * @throws FileAlreadyExistsException If file src already exists
-   * @throws FileNotFoundException If parent of src does not exist
-   *   and createParent is false
-   * @throws ParentNotDirectoryException If parent of src is not a
-   *   directory.
+   * @throws org.apache.hadoop.fs.

[23/50] [abbrv] hadoop git commit: YARN-3026. Move application-specific container allocation logic from LeafQueue to FiCaSchedulerApp. Contributed by Wangda Tan

2015-07-28 Thread jianhe
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83fe34ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index dfeb30f..c660fcb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -24,6 +24,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang.mutable.MutableObject;
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -39,6 +40,9 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
 import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
 import 
org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
 import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import 
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
 import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
 import 
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
@@ -48,11 +52,22 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManage
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
 import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment;
 import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityHeadroomProvider;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import com.google.common.annotations.VisibleForTesting;
 
 /**
  * Represents an application attempt from the viewpoint of the FIFO or Capacity
@@ -61,14 +76,22 @@ import 
org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
 @Private
 @Unstable
 public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
-
   private static final Log LOG = LogFactory.getLog(FiCaSchedulerApp.class);
 
+  static final CSAssignment NULL_ASSIGNMENT =
+  new CSAssignment(Resources.createResource(0, 0), NodeType.NODE_LOCAL);
+
+  static final CSAssignment SKIP_ASSIGNMENT = new CSAssignment(true);
+
   private final Set containersToPreempt =
 new HashSet();
 
   private CapacityHeadroomProvider headroomProvider;
 
+  private ResourceCalculator rc = new DefaultResourceCalculator();
+
+  private ResourceScheduler scheduler;
+
   public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId, 
   String user, Queue queue, ActiveUsersManager activeUsersManager,
   RMContext rmContext) {
@@ -95,6 +118,12 @@ public class FiCaSchedulerApp extends 
SchedulerApplicationAttempt {
 
 setAMResource(am

[45/50] [abbrv] hadoop git commit: HADOOP-12175. FsShell must load SpanReceierHost to support tracing (Masatake Iwasaki via Colin P. McCabe)

2015-07-28 Thread jianhe
HADOOP-12175. FsShell must load SpanReceierHost to support tracing (Masatake 
Iwasaki via Colin P. McCabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69b09573
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69b09573
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69b09573

Branch: refs/heads/YARN-1197
Commit: 69b095730bd87706ddcbabeea6c513290ee4cdf5
Parents: f170934
Author: Colin Patrick Mccabe 
Authored: Tue Jul 28 12:49:38 2015 -0700
Committer: Colin Patrick Mccabe 
Committed: Tue Jul 28 12:56:15 2015 -0700

--
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../main/java/org/apache/hadoop/fs/FsShell.java | 11 -
 .../java/org/apache/hadoop/fs/TestFsShell.java  | 24 
 .../apache/hadoop/tracing/SetSpanReceiver.java  |  2 +-
 4 files changed, 38 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b09573/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index aeaa5b9..84e7076 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1020,6 +1020,9 @@ Release 2.8.0 - UNRELEASED
 HADOOP-12245. References to misspelled REMAINING_QUATA in
 FileSystemShell.md. (Gabor Liptak via aajisaka)
 
+HADOOP-12175. FsShell must load SpanReceierHost to support tracing
+(Masatake Iwasaki via Colin P. McCabe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b09573/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index 24b6339..e9c2f73 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.shell.Command;
 import org.apache.hadoop.fs.shell.CommandFactory;
 import org.apache.hadoop.fs.shell.FsCommand;
+import org.apache.hadoop.tracing.SpanReceiverHost;
 import org.apache.hadoop.tools.TableListing;
 import org.apache.hadoop.tracing.TraceUtils;
 import org.apache.hadoop.util.Tool;
@@ -57,6 +58,9 @@ public class FsShell extends Configured implements Tool {
   private final String usagePrefix =
 "Usage: hadoop fs [generic options]";
 
+  private SpanReceiverHost spanReceiverHost;
+  static final String SEHLL_HTRACE_PREFIX = "dfs.shell.htrace.";
+
   /**
* Default ctor with no configuration.  Be sure to invoke
* {@link #setConf(Configuration)} with a valid configuration prior
@@ -97,6 +101,8 @@ public class FsShell extends Configured implements Tool {
   commandFactory.addObject(new Usage(), "-usage");
   registerCommands(commandFactory);
 }
+this.spanReceiverHost =
+SpanReceiverHost.get(getConf(), SEHLL_HTRACE_PREFIX);
   }
 
   protected void registerCommands(CommandFactory factory) {
@@ -279,7 +285,7 @@ public class FsShell extends Configured implements Tool {
 // initialize FsShell
 init();
 traceSampler = new SamplerBuilder(TraceUtils.
-wrapHadoopConf("dfs.shell.htrace.", getConf())).build();
+wrapHadoopConf(SEHLL_HTRACE_PREFIX, getConf())).build();
 int exitCode = -1;
 if (argv.length < 1) {
   printUsage(System.err);
@@ -335,6 +341,9 @@ public class FsShell extends Configured implements Tool {
   fs.close();
   fs = null;
 }
+if (this.spanReceiverHost != null) {
+  this.spanReceiverHost.closeReceivers();
+}
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b09573/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
index 12c8bcf..38e768f 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
@@ -18,6 +18,12 @@
 package org.apache.hadoop.fs;
 
 import junit.framework.AssertionFailedError;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop

[39/50] [abbrv] hadoop git commit: YARN-3853. Add docker container runtime support to LinuxContainterExecutor. Contributed by Sidharta Seethana.

2015-07-28 Thread jianhe
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
new file mode 100644
index 000..f9a890e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -0,0 +1,107 @@
+/*
+ * *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * /
+ */
+
+package 
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+
+import org.apache.hadoop.util.StringUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DockerRunCommand extends DockerCommand {
+  private static final String RUN_COMMAND = "run";
+  private final String image;
+  private List overrrideCommandWithArgs;
+
+  /** The following are mandatory: */
+  public DockerRunCommand(String containerId, String user, String image) {
+super(RUN_COMMAND);
+super.addCommandArguments("--name=" + containerId, "--user=" + user);
+this.image = image;
+  }
+
+  public DockerRunCommand removeContainerOnExit() {
+super.addCommandArguments("--rm");
+return this;
+  }
+
+  public DockerRunCommand detachOnRun() {
+super.addCommandArguments("-d");
+return this;
+  }
+
+  public DockerRunCommand setContainerWorkDir(String workdir) {
+super.addCommandArguments("--workdir=" + workdir);
+return this;
+  }
+
+  public DockerRunCommand setNetworkType(String type) {
+super.addCommandArguments("--net=" + type);
+return this;
+  }
+
+  public DockerRunCommand addMountLocation(String sourcePath, String
+  destinationPath) {
+super.addCommandArguments("-v", sourcePath + ":" + destinationPath);
+return this;
+  }
+
+  public DockerRunCommand setCGroupParent(String parentPath) {
+super.addCommandArguments("--cgroup-parent=" + parentPath);
+return this;
+  }
+
+  public DockerRunCommand addDevice(String sourceDevice, String
+  destinationDevice) {
+super.addCommandArguments("--device=" + sourceDevice + ":" +
+destinationDevice);
+return this;
+  }
+
+  public DockerRunCommand enableDetach() {
+super.addCommandArguments("--detach=true");
+return this;
+  }
+
+  public DockerRunCommand disableDetach() {
+super.addCommandArguments("--detach=false");
+return this;
+  }
+
+  public DockerRunCommand setOverrideCommandWithArgs(
+  List overrideCommandWithArgs) {
+this.overrrideCommandWithArgs = overrideCommandWithArgs;
+return this;
+  }
+
+  @Override
+  public String getCommandWithArguments() {
+List argList = new ArrayList<>();
+
+argList.add(super.getCommandWithArguments());
+argList.add(image);
+
+if (overrrideCommandWithArgs != null) {
+  argList.addAll(overrrideCommandWithArgs);
+}
+
+return StringUtils.join(" ", argList);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerExecutionException.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerExecutionException.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containe

[05/50] [abbrv] hadoop git commit: HDFS-8797. WebHdfsFileSystem creates too many connections for pread. Contributed by Jing Zhao.

2015-07-28 Thread jianhe
HDFS-8797. WebHdfsFileSystem creates too many connections for pread. 
Contributed by Jing Zhao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e91ccfad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e91ccfad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e91ccfad

Branch: refs/heads/YARN-1197
Commit: e91ccfad07ec5b5674a84009772dd31a82b4e4de
Parents: 06e5dd2
Author: Jing Zhao 
Authored: Wed Jul 22 17:42:31 2015 -0700
Committer: Jing Zhao 
Committed: Wed Jul 22 17:42:31 2015 -0700

--
 .../hadoop/hdfs/web/ByteRangeInputStream.java   | 57 +---
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  2 +
 .../hdfs/web/TestByteRangeInputStream.java  | 35 ++--
 .../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 41 ++
 4 files changed, 113 insertions(+), 22 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e91ccfad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java
index 395c9f6..bb581db 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.hdfs.web;
 
+import java.io.EOFException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.HttpURLConnection;
@@ -65,6 +66,16 @@ public abstract class ByteRangeInputStream extends 
FSInputStream {
 final boolean resolved) throws IOException;
   }
 
+  static class InputStreamAndFileLength {
+final Long length;
+final InputStream in;
+
+InputStreamAndFileLength(Long length, InputStream in) {
+  this.length = length;
+  this.in = in;
+}
+  }
+
   enum StreamStatus {
 NORMAL, SEEK, CLOSED
   }
@@ -101,7 +112,9 @@ public abstract class ByteRangeInputStream extends 
FSInputStream {
 if (in != null) {
   in.close();
 }
-in = openInputStream();
+InputStreamAndFileLength fin = openInputStream(startPos);
+in = fin.in;
+fileLength = fin.length;
 status = StreamStatus.NORMAL;
 break;
   case CLOSED:
@@ -111,20 +124,22 @@ public abstract class ByteRangeInputStream extends 
FSInputStream {
   }
 
   @VisibleForTesting
-  protected InputStream openInputStream() throws IOException {
+  protected InputStreamAndFileLength openInputStream(long startOffset)
+  throws IOException {
 // Use the original url if no resolved url exists, eg. if
 // it's the first time a request is made.
 final boolean resolved = resolvedURL.getURL() != null;
 final URLOpener opener = resolved? resolvedURL: originalURL;
 
-final HttpURLConnection connection = opener.connect(startPos, resolved);
+final HttpURLConnection connection = opener.connect(startOffset, resolved);
 resolvedURL.setURL(getResolvedUrl(connection));
 
 InputStream in = connection.getInputStream();
+final Long length;
 final Map> headers = connection.getHeaderFields();
 if (isChunkedTransferEncoding(headers)) {
   // file length is not known
-  fileLength = null;
+  length = null;
 } else {
   // for non-chunked transfer-encoding, get content-length
   final String cl = connection.getHeaderField(HttpHeaders.CONTENT_LENGTH);
@@ -133,14 +148,14 @@ public abstract class ByteRangeInputStream extends 
FSInputStream {
 + headers);
   }
   final long streamlength = Long.parseLong(cl);
-  fileLength = startPos + streamlength;
+  length = startOffset + streamlength;
 
   // Java has a bug with >2GB request streams.  It won't bounds check
   // the reads so the transfer blocks until the server times out
   in = new BoundedInputStream(in, streamlength);
 }
 
-return in;
+return new InputStreamAndFileLength(length, in);
   }
 
   private static boolean isChunkedTransferEncoding(
@@ -204,6 +219,36 @@ public abstract class ByteRangeInputStream extends 
FSInputStream {
 }
   }
 
+  @Override
+  public int read(long position, byte[] buffer, int offset, int length)
+  throws IOException {
+try (InputStream in = openInputStream(position).in) {
+  return in.read(buffer, offset, length);
+}
+  }
+
+  @Override
+  public void readFully(long position, byte[] buffer, int offset, int length)
+  throws IOException {
+final InputStreamAndFile

[34/50] [abbrv] hadoop git commit: YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. (Jonathan Yaniv and Ishai Menache via curino)

2015-07-28 Thread jianhe
YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. 
(Jonathan Yaniv and Ishai Menache via curino)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/156f24ea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/156f24ea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/156f24ea

Branch: refs/heads/YARN-1197
Commit: 156f24ead00436faad5d4aeef327a546392cd265
Parents: adcf5dd
Author: ccurino 
Authored: Sat Jul 25 07:39:47 2015 -0700
Committer: ccurino 
Committed: Sat Jul 25 07:39:47 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |   3 +
 .../reservation/AbstractReservationSystem.java  |   2 +
 .../reservation/GreedyReservationAgent.java | 390 -
 .../reservation/InMemoryPlan.java   |  13 +-
 .../InMemoryReservationAllocation.java  |   8 +-
 .../resourcemanager/reservation/Plan.java   |   1 +
 .../reservation/PlanContext.java|   2 +
 .../resourcemanager/reservation/PlanView.java   |  31 +-
 .../resourcemanager/reservation/Planner.java|  47 --
 .../RLESparseResourceAllocation.java|  55 +-
 .../reservation/ReservationAgent.java   |  72 --
 .../ReservationSchedulerConfiguration.java  |   6 +-
 .../reservation/ReservationSystem.java  |   5 +-
 .../reservation/ReservationSystemUtil.java  |   6 +-
 .../reservation/SimpleCapacityReplanner.java| 113 ---
 .../planning/AlignedPlannerWithGreedy.java  | 123 +++
 .../planning/GreedyReservationAgent.java|  97 +++
 .../reservation/planning/IterativePlanner.java  | 338 
 .../reservation/planning/Planner.java   |  49 ++
 .../reservation/planning/PlanningAlgorithm.java | 207 +
 .../reservation/planning/ReservationAgent.java  |  73 ++
 .../planning/SimpleCapacityReplanner.java   | 118 +++
 .../reservation/planning/StageAllocator.java|  55 ++
 .../planning/StageAllocatorGreedy.java  | 152 
 .../planning/StageAllocatorLowCostAligned.java  | 360 
 .../planning/StageEarliestStart.java|  46 ++
 .../planning/StageEarliestStartByDemand.java| 106 +++
 .../StageEarliestStartByJobArrival.java |  39 +
 .../planning/TryManyReservationAgents.java  | 114 +++
 .../reservation/ReservationSystemTestUtil.java  |   5 +-
 .../reservation/TestCapacityOverTimePolicy.java |   2 +-
 .../TestCapacitySchedulerPlanFollower.java  |   1 +
 .../reservation/TestFairReservationSystem.java  |   1 -
 .../TestFairSchedulerPlanFollower.java  |   1 +
 .../reservation/TestGreedyReservationAgent.java | 604 --
 .../reservation/TestInMemoryPlan.java   |   2 +
 .../reservation/TestNoOverCommitPolicy.java |   1 +
 .../TestRLESparseResourceAllocation.java|  51 +-
 .../TestSchedulerPlanFollowerBase.java  |   1 +
 .../TestSimpleCapacityReplanner.java| 162 
 .../planning/TestAlignedPlanner.java| 820 +++
 .../planning/TestGreedyReservationAgent.java| 611 ++
 .../planning/TestSimpleCapacityReplanner.java   | 170 
 43 files changed, 3634 insertions(+), 1429 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/156f24ea/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 55258a6..883d009 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -147,6 +147,9 @@ Release 2.8.0 - UNRELEASED
 YARN-2019. Retrospect on decision of making RM crashed if any exception 
throw 
 in ZKRMStateStore. (Jian He via junping_du)
 
+YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. 
+(Jonathan Yaniv and Ishai Menache via curino)
+
   IMPROVEMENTS
 
 YARN-644. Basic null check is not performed on passed in arguments before

http://git-wip-us.apache.org/repos/asf/hadoop/blob/156f24ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
index 8a15ac6..d2603c1 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/serv

[33/50] [abbrv] hadoop git commit: YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. (Jonathan Yaniv and Ishai Menache via curino)

2015-07-28 Thread jianhe
http://git-wip-us.apache.org/repos/asf/hadoop/blob/156f24ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
new file mode 100644
index 000..9a0a0f0
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
@@ -0,0 +1,207 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryReservationAllocation;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ContractValidationException;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+
+/**
+ * An abstract class that follows the general behavior of planning algorithms.
+ */
+public abstract class PlanningAlgorithm implements ReservationAgent {
+
+  /**
+   * Performs the actual allocation for a ReservationDefinition within a Plan.
+   *
+   * @param reservationId the identifier of the reservation
+   * @param user the user who owns the reservation
+   * @param plan the Plan to which the reservation must be fitted
+   * @param contract encapsulates the resources required by the user for his
+   *  session
+   * @param oldReservation the existing reservation (null if none)
+   * @return whether the allocateUser function was successful or not
+   *
+   * @throws PlanningException if the session cannot be fitted into the plan
+   * @throws ContractValidationException
+   */
+  protected boolean allocateUser(ReservationId reservationId, String user,
+  Plan plan, ReservationDefinition contract,
+  ReservationAllocation oldReservation) throws PlanningException,
+  ContractValidationException {
+
+// Adjust the ResourceDefinition to account for system "imperfections"
+// (e.g., scheduling delays for large containers).
+ReservationDefinition adjustedContract = adjustContract(plan, contract);
+
+// Compute the job allocation
+RLESparseResourceAllocation allocation =
+computeJobAllocation(plan, reservationId, adjustedContract);
+
+// If no job allocation was found, fail
+if (allocation == null) {
+  throw new PlanningException(
+  "The planning algorithm could not find a valid allocation"
+  + " for your request");
+}
+
+// Translate the allocation to a map (with zero paddings)
+long step = plan.getStep();
+long jobArrival = stepRoundUp(adjustedContract.getArrival(), step);
+long jobDeadline = stepRoundUp(adjustedContract.getDeadline(), step);
+Map mapAllocations =
+allocationsToPaddedMap(allocation, jobArrival, jobDeadline);
+
+// Create the reservation
+ReservationAllocation capReservation =
+new InMemoryReservationAllocation(reservationId, // ID
+adjustedContract, // Contract
+user, // User name
+plan.getQueueName()

[49/50] [abbrv] hadoop git commit: YARN-1645. ContainerManager implementation to support container resizing. Contributed by Meng Ding & Wangda Tan

2015-07-28 Thread jianhe
YARN-1645. ContainerManager implementation to support container resizing. 
Contributed by Meng Ding & Wangda Tan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4facdcec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4facdcec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4facdcec

Branch: refs/heads/YARN-1197
Commit: 4facdcec910a86e043a065ee26ca73f6bbd836f0
Parents: b3f2830
Author: Jian He 
Authored: Tue Jul 21 16:10:40 2015 -0700
Committer: Jian He 
Committed: Tue Jul 28 13:53:32 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |   3 +
 .../CMgrDecreaseContainersResourceEvent.java|  37 
 .../nodemanager/ContainerManagerEventType.java  |   1 +
 .../containermanager/ContainerManagerImpl.java  | 180 --
 .../container/ChangeContainerResourceEvent.java |  36 
 .../container/ContainerEventType.java   |   4 +
 .../nodemanager/DummyContainerManager.java  |   6 +-
 .../TestContainerManagerWithLCE.java|  22 +++
 .../BaseContainerManagerTest.java   |  43 -
 .../containermanager/TestContainerManager.java  | 190 ++-
 10 files changed, 486 insertions(+), 36 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/4facdcec/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b09362e..eace759 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -162,6 +162,9 @@ Release 2.8.0 - UNRELEASED
 YARN-1449. AM-NM protocol changes to support container resizing.
 (Meng Ding & Wangda Tan via jianhe)
 
+YARN-1645. ContainerManager implementation to support container resizing.
+(Meng Ding & Wangda Tan via jianhe)
+
   IMPROVEMENTS
 
 YARN-644. Basic null check is not performed on passed in arguments before

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4facdcec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
new file mode 100644
index 000..9479d0b
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import java.util.List;
+
+public class CMgrDecreaseContainersResourceEvent extends ContainerManagerEvent 
{
+
+  private final List containersToDecrease;
+
+  public CMgrDecreaseContainersResourceEvent(List
+  containersToDecrease) {
+super(ContainerManagerEventType.DECREASE_CONTAINERS_RESOURCE);
+this.containersToDecrease = containersToDecrease;
+  }
+
+  public List getContainersToDecrease() {
+return this.containersToDecrease;
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/4facdcec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
 
b/hadoop-yarn-project/hadoop-yarn/h

[31/50] [abbrv] hadoop git commit: YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. (Jonathan Yaniv and Ishai Menache via curino)

2015-07-28 Thread jianhe
http://git-wip-us.apache.org/repos/asf/hadoop/blob/156f24ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
new file mode 100644
index 000..bd18a2f
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
@@ -0,0 +1,611 @@
+/***
+ *   Licensed to the Apache Software Foundation (ASF) under one
+ *   or more contributor license agreements.  See the NOTICE file
+ *   distributed with this work for additional information
+ *   regarding copyright ownership.  The ASF licenses this file
+ *   to you under the Apache License, Version 2.0 (the
+ *   "License"); you may not use this file except in compliance
+ *   with the License.  You may obtain a copy of the License at
+ *  
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *  
+ *   Unless required by applicable law or agreed to in writing, software
+ *   distributed under the License is distributed on an "AS IS" BASIS,
+ *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *   See the License for the specific language governing permissions and
+ *   limitations under the License.
+ 
***/
+package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacityOverTimePolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryPlan;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryReservationAllocation;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil;
+import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Before;
+import org.junit.Test;
+import org.mortbay.log.Log;
+
+public class TestGreedyReservationAgent {
+
+  ReservationAgent agent;
+  InMemoryPlan plan;
+  Resource minAlloc = Resource.newInstance(1024, 1);
+  ResourceCalculator res = new DefaultResourceCalculator();
+  Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
+  Random rand = new Random();
+  long step;
+
+  @Before
+  public void setup() throws Exception {
+
+long seed = rand.nextLong();
+rand.setSeed(seed);
+Log.info("Running with seed: " + seed);
+
+// setting completely loose quotas
+long timeWindow = 100L;
+Resource clusterCapacity = Resource.newInstance(100 * 1024, 100);
+step = 1000L;
+ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
+String reservationQ = testUtil.getFullReservationQueueName();
+
+float instConstraint = 100;
+float a

[26/50] [abbrv] hadoop git commit: YARN-3973. Recent changes to application priority management break reservation system from YARN-1051 (Carlo Curino via wangda)

2015-07-28 Thread jianhe
YARN-3973. Recent changes to application priority management break reservation 
system from YARN-1051 (Carlo Curino via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3bd7b4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3bd7b4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3bd7b4a

Branch: refs/heads/YARN-1197
Commit: a3bd7b4a59b3664273dc424f240356838213d4e7
Parents: ff9c13e
Author: Wangda Tan 
Authored: Fri Jul 24 16:44:18 2015 -0700
Committer: Wangda Tan 
Committed: Fri Jul 24 16:44:18 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt| 6 +-
 .../resourcemanager/scheduler/capacity/CapacityScheduler.java  | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3bd7b4a/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c295784..55258a6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -667,7 +667,8 @@ Release 2.8.0 - UNRELEASED
 
 YARN-3956. Fix TestNodeManagerHardwareUtils fails on Mac (Varun Vasudev 
via wangda)
 
-YARN-3941. Proportional Preemption policy should try to avoid sending 
duplicate PREEMPT_CONTAINER event to scheduler. (Sunil G via wangda)
+YARN-3941. Proportional Preemption policy should try to avoid sending 
duplicate 
+PREEMPT_CONTAINER event to scheduler. (Sunil G via wangda)
 
 YARN-3900. Protobuf layout of yarn_security_token causes errors in other 
protos
 that include it (adhoot via rkanter)
@@ -678,6 +679,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3957. FairScheduler NPE In FairSchedulerQueueInfo causing scheduler 
page to 
 return 500. (Anubhav Dhoot via kasha)
 
+YARN-3973. Recent changes to application priority management break 
+reservation system from YARN-1051. (Carlo Curino via wangda)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3bd7b4a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 68e608a..0b39d35 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1867,7 +1867,7 @@ public class CapacityScheduler extends
 
   private Priority getDefaultPriorityForQueue(String queueName) {
 Queue queue = getQueue(queueName);
-if (null == queue) {
+if (null == queue || null == queue.getDefaultApplicationPriority()) {
   // Return with default application priority
   return Priority.newInstance(CapacitySchedulerConfiguration
   .DEFAULT_CONFIGURATION_APPLICATION_PRIORITY);



[16/50] [abbrv] hadoop git commit: HDFS-8806. Inconsistent metrics: number of missing blocks with replication factor 1 not properly cleared. Contributed by Zhe Zhang.

2015-07-28 Thread jianhe
HDFS-8806. Inconsistent metrics: number of missing blocks with replication 
factor 1 not properly cleared. Contributed by Zhe Zhang.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/206d4933
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/206d4933
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/206d4933

Branch: refs/heads/YARN-1197
Commit: 206d4933a567147b62f463c2daa3d063ad40822b
Parents: e202efa
Author: Akira Ajisaka 
Authored: Fri Jul 24 18:28:44 2015 +0900
Committer: Akira Ajisaka 
Committed: Fri Jul 24 18:28:44 2015 +0900

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt   | 3 +++
 .../hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java | 3 ++-
 2 files changed, 5 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/206d4933/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f86d41e..b348a5a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1097,6 +1097,9 @@ Release 2.7.2 - UNRELEASED
 HDFS-6945. BlockManager should remove a block from excessReplicateMap and
 decrement ExcessBlocks metric when the block is removed. (aajisaka)
 
+HDFS-8806. Inconsistent metrics: number of missing blocks with replication
+factor 1 not properly cleared. (Zhe Zhang via aajisaka)
+
 Release 2.7.1 - 2015-07-06
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/206d4933/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
index d8aec99..128aae6 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
@@ -101,10 +101,11 @@ class UnderReplicatedBlocks implements 
Iterable {
   /**
* Empty the queues and timestamps.
*/
-  void clear() {
+  synchronized void clear() {
 for (int i = 0; i < LEVEL; i++) {
   priorityQueues.get(i).clear();
 }
+corruptReplOneBlocks = 0;
 timestampsMap.clear();
   }
 



[18/50] [abbrv] hadoop git commit: YARN-3967. Fetch the application report from the AHS if the RM does not know about it. Contributed by Mit Desai

2015-07-28 Thread jianhe
YARN-3967. Fetch the application report from the AHS if the RM does not
know about it. Contributed by Mit Desai


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbd60632
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbd60632
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbd60632

Branch: refs/heads/YARN-1197
Commit: fbd6063269221ec25834684477f434e19f0b66af
Parents: ee233ec
Author: Xuan 
Authored: Fri Jul 24 10:15:54 2015 -0700
Committer: Xuan 
Committed: Fri Jul 24 10:15:54 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |   3 +
 .../yarn/server/webproxy/AppReportFetcher.java  |  79 +++--
 .../server/webproxy/TestAppReportFetcher.java   | 117 +++
 3 files changed, 187 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbd60632/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8bc9e4c..a25387d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -683,6 +683,9 @@ Release 2.7.2 - UNRELEASED
 YARN-3170. YARN architecture document needs updating. (Brahma Reddy Battula
 via ozawa)
 
+YARN-3967. Fetch the application report from the AHS if the RM does not 
know about it.
+(Mit Desai via xgong)
+
   OPTIMIZATIONS
 
   BUG FIXES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbd60632/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
index 5c93413..6aa43eb 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
@@ -24,11 +24,15 @@ import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.ipc.RPC;
 import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
 import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.client.AHSProxy;
 import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
 import org.apache.hadoop.yarn.exceptions.YarnException;
 import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
 import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -41,38 +45,73 @@ public class AppReportFetcher {
   private static final Log LOG = LogFactory.getLog(AppReportFetcher.class);
   private final Configuration conf;
   private final ApplicationClientProtocol applicationsManager;
+  private final ApplicationHistoryProtocol historyManager;
   private final RecordFactory recordFactory = 
RecordFactoryProvider.getRecordFactory(null);
-  
+  private boolean isAHSEnabled;
+
   /**
-   * Create a new Connection to the RM to fetch Application reports.
+   * Create a new Connection to the RM/Application History Server
+   * to fetch Application reports.
* @param conf the conf to use to know where the RM is.
*/
   public AppReportFetcher(Configuration conf) {
+if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
+  isAHSEnabled = true;
+}
 this.conf = conf;
 try {
   applicationsManager = ClientRMProxy.createRMProxy(conf,
   ApplicationClientProtocol.class);
+  if (isAHSEnabled) {
+historyManager = getAHSProxy(conf);
+  } else {
+this.historyManager = null;
+  }
 } catch (IOException e) {
   throw new YarnRuntimeException(e);
 }
   }
   
   /**
-   * Just call directly into the applicationsManager given instead of creating
-   * a remote connection to it.  This is mostly for when the Proxy is running
-   * as part of

[29/50] [abbrv] hadoop git commit: HADOOP-12202. releasedocmaker drops missing component and assignee entries (aw)

2015-07-28 Thread jianhe
HADOOP-12202. releasedocmaker drops missing component and assignee entries (aw)

(cherry picked from commit adbacf7010373dbe6df239688b4cebd4a93a69e4)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d7697831
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d7697831
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d7697831

Branch: refs/heads/YARN-1197
Commit: d7697831e3b24bec149990feed819e7d96f78184
Parents: e8b62d1
Author: Allen Wittenauer 
Authored: Tue Jul 7 14:30:32 2015 -0700
Committer: Andrew Wang 
Committed: Fri Jul 24 18:31:44 2015 -0700

--
 dev-support/releasedocmaker.py | 24 
 1 file changed, 12 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7697831/dev-support/releasedocmaker.py
--
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index 6e01260..409d8e3 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -420,6 +420,8 @@ def main():
   else:
 title=options.title
 
+  haderrors=False
+
   for v in versions:
 vstr=str(v)
 jlist = JiraIter(vstr,projects)
@@ -468,14 +470,6 @@ def main():
 for jira in sorted(jlist):
   if jira.getIncompatibleChange():
 incompatlist.append(jira)
-if (len(jira.getReleaseNote())==0):
-warningCount+=1
-
-  if jira.checkVersionString():
- warningCount+=1
-
-  if jira.checkMissingComponent() or jira.checkMissingAssignee():
-errorCount+=1
   elif jira.getType() == "Bug":
 buglist.append(jira)
   elif jira.getType() == "Improvement":
@@ -496,6 +490,7 @@ def main():
  notableclean(jira.getSummary()))
 
   if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
+warningCount+=1
 reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
 reloutputs.writeKeyRaw(jira.getProject(), line)
 line ='\n**WARNING: No release note provided for this incompatible 
change.**\n\n'
@@ -503,9 +498,11 @@ def main():
 reloutputs.writeKeyRaw(jira.getProject(), line)
 
   if jira.checkVersionString():
+  warningCount+=1
   lintMessage += "\nWARNING: Version string problem for %s " % 
jira.getId()
 
   if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
+  errorCount+=1
   errorMessage=[]
   jira.checkMissingComponent() and errorMessage.append("component")
   jira.checkMissingAssignee() and errorMessage.append("assignee")
@@ -520,11 +517,11 @@ def main():
 if (options.lint is True):
 print lintMessage
 print "==="
-print "Error:%d, Warning:%d \n" % (errorCount, warningCount)
-
+print "%s: Error:%d, Warning:%d \n" % (vstr, errorCount, warningCount)
 if (errorCount>0):
-cleanOutputDir(version)
-sys.exit(1)
+   haderrors=True
+   cleanOutputDir(vstr)
+   continue
 
 reloutputs.writeAll("\n\n")
 reloutputs.close()
@@ -571,5 +568,8 @@ def main():
   if options.index:
 buildindex(title,options.license)
 
+  if haderrors is True:
+sys.exit(1)
+
 if __name__ == "__main__":
   main()



[44/50] [abbrv] hadoop git commit: YARN-3982. container-executor parsing of container-executor.cfg broken in trunk and branch-2. Contributed by Varun Vasudev

2015-07-28 Thread jianhe
YARN-3982. container-executor parsing of container-executor.cfg broken
in trunk and branch-2. Contributed by Varun Vasudev


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f1709342
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f1709342
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f1709342

Branch: refs/heads/YARN-1197
Commit: f17093421521efcbdc813f6f2b8411e45ecc7863
Parents: 030fcfa
Author: Xuan 
Authored: Mon Jul 27 23:45:58 2015 -0700
Committer: Xuan 
Committed: Mon Jul 27 23:45:58 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |  3 +++
 .../container-executor/impl/configuration.c |  4 ++--
 .../test/test-container-executor.c  | 22 +---
 3 files changed, 20 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4f8484a..b4666e8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -698,6 +698,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3846. RM Web UI queue filter is not working for sub queue.
 (Mohammad Shahid Khan via jianhe)
 
+YARN-3982. container-executor parsing of container-executor.cfg broken in
+trunk and branch-2. (Varun Vasudev via xgong)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index 2825367..373dbfd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -284,11 +284,11 @@ char * get_value(const char* key) {
 
 /**
  * Function to return an array of values for a key.
- * Value delimiter is assumed to be a '%'.
+ * Value delimiter is assumed to be a ','.
  */
 char ** get_values(const char * key) {
   char *value = get_value(key);
-  return extract_values(value);
+  return extract_values_delim(value, ",");
 }
 
 char ** extract_values_delim(char *value, const char *delim) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 99bcf34..001a37d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -145,7 +145,7 @@ void check_pid_file(const char* pid_file, pid_t mypid) {
   }
 
   char myPidBuf[33];
-  snprintf(myPidBuf, 33, "%" PRId64, (int64_t)mypid);
+  snprintf(myPidBuf, 33, "%" PRId64, (int64_t)(mypid + 1));
   if (strncmp(pidBuf, myPidBuf, strlen(myPidBuf)) != 0) {
 printf("FAIL: failed to find matching pid in pid file\n");
 printf("FAIL: Expected pid %" PRId64 " : Got %.*s", (int64_t)mypid,
@@ -212,15 +212,15 @@ void test_get_app_log_dir() {
   free(logdir);
 }
 
-void test_check_user() {
+void test_check_user(int expectedFailure) {
   printf("\nTesting test_check_user\n");
   struct passwd *user = check_user(username);
-  if (user == NULL) {
+  if (user == NULL && !expectedFailure) {
 printf("FAIL: failed check for user %s\n", username);
 exit(1);
   }
   free(user);
-  if (check_user("lp") != NULL) {
+  if (check_user("lp") != NULL && !expectedFailure) {
 printf("FAIL: failed check for system user lp\n");
 exit(1);
   }
@@ -228,7 +228,7 @@ void test_check_user() {
 printf("FAIL: failed check for system user root\n

[08/50] [abbrv] hadoop git commit: HADOOP-12161. Add getStoragePolicy API to the FileSystem interface. (Contributed by Brahma Reddy Battula)

2015-07-28 Thread jianhe
HADOOP-12161. Add getStoragePolicy API to the FileSystem interface. 
(Contributed by Brahma Reddy Battula)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adfa34ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adfa34ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adfa34ff

Branch: refs/heads/YARN-1197
Commit: adfa34ff9992295a6d2496b259d8c483ed90b566
Parents: 3bba180
Author: Arpit Agarwal 
Authored: Thu Jul 23 10:13:04 2015 -0700
Committer: Arpit Agarwal 
Committed: Thu Jul 23 10:13:04 2015 -0700

--
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../apache/hadoop/fs/AbstractFileSystem.java| 13 +
 .../java/org/apache/hadoop/fs/FileContext.java  | 20 
 .../java/org/apache/hadoop/fs/FileSystem.java   | 13 +
 .../org/apache/hadoop/fs/FilterFileSystem.java  |  6 ++
 .../java/org/apache/hadoop/fs/FilterFs.java |  6 ++
 .../org/apache/hadoop/fs/viewfs/ChRootedFs.java |  6 ++
 .../org/apache/hadoop/fs/viewfs/ViewFs.java | 15 +++
 .../org/apache/hadoop/fs/TestHarFileSystem.java |  3 +++
 .../main/java/org/apache/hadoop/fs/Hdfs.java|  5 +
 .../java/org/apache/hadoop/hdfs/DFSClient.java  | 18 ++
 .../hadoop/hdfs/DistributedFileSystem.java  | 19 +++
 .../hadoop/hdfs/TestBlockStoragePolicy.java | 17 +
 13 files changed, 144 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index ff7d2ad..f1a3bc9 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -716,6 +716,9 @@ Release 2.8.0 - UNRELEASED
 HADOOP-12184. Remove unused Linux-specific constants in NativeIO (Martin
 Walsh via Colin P. McCabe)
 
+HADOOP-12161. Add getStoragePolicy API to the FileSystem interface.
+(Brahma Reddy Battula via Arpit Agarwal)
+
   OPTIMIZATIONS
 
 HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index cb3fb86..2bc3859 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -1237,6 +1237,19 @@ public abstract class AbstractFileSystem {
   }
 
   /**
+   * Retrieve the storage policy for a given file or directory.
+   *
+   * @param src file or directory path.
+   * @return storage policy for give file.
+   * @throws IOException
+   */
+  public BlockStoragePolicySpi getStoragePolicy(final Path src)
+  throws IOException {
+throw new UnsupportedOperationException(getClass().getSimpleName()
++ " doesn't support getStoragePolicy");
+  }
+
+  /**
* Retrieve all the storage policies supported by this file system.
*
* @return all storage policies supported by this filesystem.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 0f21a61..a98d662 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.fs.permission.FsAction;
 import org.apache.hadoop.fs.permission.FsPermission;
 import static 
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
 import static 
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
+
 import org.apache.hadoop.io.IOUtils;
 import org.apache.hadoop.ipc.RpcClientException;
 import org.apache.hadoop.ipc.RpcServerException;
@@ -2692,6 +2693,25 @@ public class FileContext {
   }
 
   /**
+   * Query the effective storage policy ID for the given file or directory.
+   *
+   * @param src file or direc

[43/50] [abbrv] hadoop git commit: HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)

2015-07-28 Thread jianhe
HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/030fcfa9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/030fcfa9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/030fcfa9

Branch: refs/heads/YARN-1197
Commit: 030fcfa99c345ad57625486eeabedebf2fd4411f
Parents: e21dde5
Author: Arun Suresh 
Authored: Mon Jul 27 23:02:03 2015 -0700
Committer: Arun Suresh 
Committed: Mon Jul 27 23:02:03 2015 -0700

--
 .../apache/hadoop/io/retry/MultiException.java  |  49 +++
 .../hadoop/io/retry/RetryInvocationHandler.java |  99 +-
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |   2 +
 .../ha/ConfiguredFailoverProxyProvider.java |  52 ++-
 .../ha/RequestHedgingProxyProvider.java | 186 ++
 .../markdown/HDFSHighAvailabilityWithNFS.md |   9 +-
 .../markdown/HDFSHighAvailabilityWithQJM.md |  10 +-
 .../ha/TestRequestHedgingProxyProvider.java | 350 +++
 8 files changed, 724 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/030fcfa9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
new file mode 100644
index 000..4963a2d
--- /dev/null
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.io.retry;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Holder class that clients can use to return multiple exceptions.
+ */
+public class MultiException extends IOException {
+
+  private final Map exes;
+
+  public MultiException(Map exes) {
+this.exes = exes;
+  }
+
+  public Map getExceptions() {
+return exes;
+  }
+
+  @Override
+  public String toString() {
+StringBuilder sb = new StringBuilder("{");
+for (Exception e : exes.values()) {
+  sb.append(e.toString()).append(", ");
+}
+sb.append("}");
+return "MultiException[" + sb.toString() + "]";
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/030fcfa9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
index 543567e..9256356 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
@@ -23,6 +23,8 @@ import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.logging.Log;
@@ -101,7 +103,7 @@ public class RetryInvocationHandler implements 
RpcInvocationHandler {
 Object ret = invokeMethod(method, args);
 hasMadeASuccessfulCall = true;
 return ret;
-  } catch (Exception e) {
+  } catch (Exception ex) {
 boolean isIdempotentOrAtMostOnce = proxyProvider.getInterface()
 .getMethod(method.getName(), method.getParameterTypes())
 .isAnnotationPresent(Idempotent.class);
@@ -110,15 +112,16 @@ public class RetryInvocationHandler implements 
RpcInvocationHandler {
   .getMethod(method.getName(), method.getParameterTypes())
   .isA

[12/50] [abbrv] hadoop git commit: HADOOP-12009: Clarify FileSystem.listStatus() sorting order & fix FileSystemContractBaseTest:testListStatus. (J.Andreina via jghoman)

2015-07-28 Thread jianhe
HADOOP-12009: Clarify FileSystem.listStatus() sorting order & fix 
FileSystemContractBaseTest:testListStatus. (J.Andreina via jghoman)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab3197c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab3197c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab3197c2

Branch: refs/heads/YARN-1197
Commit: ab3197c20452e0dd908193d6854c204e6ee34645
Parents: 1d3026e
Author: Jakob Homan 
Authored: Thu Jul 23 17:46:13 2015 -0700
Committer: Jakob Homan 
Committed: Thu Jul 23 17:46:13 2015 -0700

--
 hadoop-common-project/hadoop-common/CHANGES.txt|  3 +++
 .../main/java/org/apache/hadoop/fs/FileSystem.java | 17 -
 .../src/site/markdown/filesystem/filesystem.md |  4 
 .../hadoop/fs/FileSystemContractBaseTest.java  | 11 ---
 4 files changed, 31 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab3197c2/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 6c18add..56edcac 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -497,6 +497,9 @@ Trunk (Unreleased)
 
 HADOOP-11762. Enable swift distcp to secure HDFS (Chen He via aw)
 
+HADOOP-12009. Clarify FileSystem.listStatus() sorting order & fix
+FileSystemContractBaseTest:testListStatus. (J.Andreina via jghoman)
+
   OPTIMIZATIONS
 
 HADOOP-7761. Improve the performance of raw comparisons. (todd)

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab3197c2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index a01d3ea..8f32644 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1501,7 +1501,9 @@ public abstract class FileSystem extends Configured 
implements Closeable {
   /**
* List the statuses of the files/directories in the given path if the path 
is
* a directory.
-   * 
+   * 
+   * Does not guarantee to return the List of files/directories status in a
+   * sorted order.
* @param f given path
* @return the statuses of the files/directories in the given patch
* @throws FileNotFoundException when the path does not exist;
@@ -1543,6 +1545,9 @@ public abstract class FileSystem extends Configured 
implements Closeable {
   /**
* Filter files/directories in the given path using the user-supplied path
* filter.
+   * 
+   * Does not guarantee to return the List of files/directories status in a
+   * sorted order.
* 
* @param f
*  a path name
@@ -1563,6 +1568,9 @@ public abstract class FileSystem extends Configured 
implements Closeable {
   /**
* Filter files/directories in the given list of paths using default
* path filter.
+   * 
+   * Does not guarantee to return the List of files/directories status in a
+   * sorted order.
* 
* @param files
*  a list of paths
@@ -1579,6 +1587,9 @@ public abstract class FileSystem extends Configured 
implements Closeable {
   /**
* Filter files/directories in the given list of paths using user-supplied
* path filter.
+   * 
+   * Does not guarantee to return the List of files/directories status in a
+   * sorted order.
* 
* @param files
*  a list of paths
@@ -1739,6 +1750,8 @@ public abstract class FileSystem extends Configured 
implements Closeable {
* while consuming the entries. Each file system implementation should
* override this method and provide a more efficient implementation, if
* possible. 
+   * Does not guarantee to return the iterator that traverses statuses
+   * of the files in a sorted order.
*
* @param p target path
* @return remote iterator
@@ -1766,6 +1779,8 @@ public abstract class FileSystem extends Configured 
implements Closeable {
 
   /**
* List the statuses and block locations of the files in the given path.
+   * Does not guarantee to return the iterator that traverses statuses
+   * of the files in a sorted order.
* 
* If the path is a directory, 
*   if recursive is false, returns files in the directory;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab3197c2/hadoop-common-project/hadoop-common/src/site/markdown/filesy

[20/50] [abbrv] hadoop git commit: YARN-3957. FairScheduler NPE In FairSchedulerQueueInfo causing scheduler page to return 500. (Anubhav Dhoot via kasha)

2015-07-28 Thread jianhe
YARN-3957. FairScheduler NPE In FairSchedulerQueueInfo causing scheduler page 
to return 500. (Anubhav Dhoot via kasha)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d19d1877
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d19d1877
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d19d1877

Branch: refs/heads/YARN-1197
Commit: d19d18775368f5aaa254881165acc1299837072b
Parents: f8f6091
Author: Karthik Kambatla 
Authored: Fri Jul 24 11:44:37 2015 -0700
Committer: Karthik Kambatla 
Committed: Fri Jul 24 11:44:37 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |  3 +
 .../webapp/dao/FairSchedulerQueueInfo.java  |  4 +-
 .../webapp/dao/TestFairSchedulerQueueInfo.java  | 59 
 3 files changed, 65 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19d1877/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a25387d..44e5510 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -672,6 +672,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3845. Scheduler page does not render RGBA color combinations in IE11. 
 (Contributed by Mohammad Shahid Khan)
 
+YARN-3957. FairScheduler NPE In FairSchedulerQueueInfo causing scheduler 
page to 
+return 500. (Anubhav Dhoot via kasha)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19d1877/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index 9b297a2..7ba0988 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -19,6 +19,7 @@
 package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
 
 
+import java.util.ArrayList;
 import java.util.Collection;
 
 import javax.xml.bind.annotation.XmlAccessType;
@@ -204,6 +205,7 @@ public class FairSchedulerQueueInfo {
   }
 
   public Collection getChildQueues() {
-return childQueues.getQueueInfoList();
+return childQueues != null ? childQueues.getQueueInfoList() :
+new ArrayList();
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19d1877/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
new file mode 100644
index 000..973afcf
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for

[41/50] [abbrv] hadoop git commit: YARN-3846. RM Web UI queue filter is not working for sub queue. Contributed by Mohammad Shahid Khan

2015-07-28 Thread jianhe
YARN-3846. RM Web UI queue filter is not working for sub queue. Contributed by 
Mohammad Shahid Khan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3572ebd7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3572ebd7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3572ebd7

Branch: refs/heads/YARN-1197
Commit: 3572ebd738aa5fa8b0906d75fb12cc6cbb991573
Parents: 3e6fce9
Author: Jian He 
Authored: Mon Jul 27 16:57:11 2015 -0700
Committer: Jian He 
Committed: Mon Jul 27 17:12:05 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt | 3 +++
 .../server/resourcemanager/webapp/CapacitySchedulerPage.java| 5 -
 2 files changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3572ebd7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 534c55a..4f8484a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -695,6 +695,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3958. TestYarnConfigurationFields should be moved to hadoop-yarn-api
 module. (Varun Saxena via aajisaka)
 
+YARN-3846. RM Web UI queue filter is not working for sub queue.
+(Mohammad Shahid Khan via jianhe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3572ebd7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 12a3013..d8971b7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -516,7 +516,10 @@ class CapacitySchedulerPage extends RmView {
   "  $('#cs').bind('select_node.jstree', function(e, data) {",
   "var q = $('.q', data.rslt.obj).first().text();",
   "if (q == 'Queue: root') q = '';",
-  "else q = '^' + q.substr(q.lastIndexOf(':') + 2) + '$';",
+  "else {",
+  "  q = q.substr(q.lastIndexOf(':') + 2);",
+  "  q = '^' + q.substr(q.lastIndexOf('.') + 1) + '$';",
+  "}",
   "$('#apps').dataTable().fnFilter(q, 4, true);",
   "  });",
   "  $('#cs').show();",



[48/50] [abbrv] hadoop git commit: YARN-3866. AM-RM protocol changes to support container resizing. Contributed by Meng Ding

2015-07-28 Thread jianhe
YARN-3866. AM-RM protocol changes to support container resizing. Contributed by 
Meng Ding


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7758afd3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7758afd3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7758afd3

Branch: refs/heads/YARN-1197
Commit: 7758afd38469a7f265d5b48265b1290f75ee6b33
Parents: 69b0957
Author: Jian He 
Authored: Mon Jul 13 17:34:26 2015 -0700
Committer: Jian He 
Committed: Tue Jul 28 13:53:32 2015 -0700

--
 .../app/local/TestLocalContainerAllocator.java  |   6 +-
 hadoop-yarn-project/CHANGES.txt |   2 +
 .../api/protocolrecords/AllocateRequest.java|  57 --
 .../api/protocolrecords/AllocateResponse.java   |  42 +++--
 .../records/ContainerResourceChangeRequest.java | 117 +
 .../api/records/ContainerResourceDecrease.java  |  78 -
 .../api/records/ContainerResourceIncrease.java  |  84 -
 .../ContainerResourceIncreaseRequest.java   |  80 -
 .../yarn/api/records/ContainerStatus.java   |  13 ++
 .../src/main/proto/yarn_protos.proto|  14 +-
 .../src/main/proto/yarn_service_protos.proto|  16 +-
 .../impl/pb/AllocateRequestPBImpl.java  | 119 ++---
 .../impl/pb/AllocateResponsePBImpl.java | 175 +--
 .../ContainerResourceChangeRequestPBImpl.java   | 141 +++
 .../pb/ContainerResourceDecreasePBImpl.java | 136 --
 .../pb/ContainerResourceIncreasePBImpl.java | 171 --
 .../ContainerResourceIncreaseRequestPBImpl.java | 141 ---
 .../records/impl/pb/ContainerStatusPBImpl.java  |  31 +++-
 .../hadoop/yarn/api/TestAllocateRequest.java|  73 
 .../hadoop/yarn/api/TestAllocateResponse.java   | 114 
 .../yarn/api/TestContainerResourceDecrease.java |  66 ---
 .../yarn/api/TestContainerResourceIncrease.java |  74 
 .../TestContainerResourceIncreaseRequest.java   |  68 ---
 .../hadoop/yarn/api/TestPBImplRecords.java  |  34 +---
 24 files changed, 533 insertions(+), 1319 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7758afd3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
index f901ed8..167d804 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
@@ -46,8 +46,6 @@ import 
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
 import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
 import org.apache.hadoop.yarn.api.records.ApplicationId;
 import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncrease;
 import org.apache.hadoop.yarn.api.records.ContainerStatus;
 import org.apache.hadoop.yarn.api.records.NMToken;
 import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -254,8 +252,8 @@ public class TestLocalContainerAllocator {
   Resources.none(), null, 1, null,
   Collections.emptyList(),
   yarnToken,
-  Collections.emptyList(),
-  Collections.emptyList());
+  Collections.emptyList(),
+  Collections.emptyList());
 }
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7758afd3/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b4666e8..87f0e91 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -156,6 +156,8 @@ Release 2.8.0 - UNRELEASED
 YARN-3853. Add docker container runtime support to LinuxContainterExecutor.
 (Sidharta Seethana via vvasudev)
 
+YARN-3866. AM-RM protocol changes to support container resizing. (Meng Ding
+via jianhe)
 
   IMPROVEMENTS
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/7758afd3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/mai

[50/50] [abbrv] hadoop git commit: YARN-3867. ContainerImpl changes to support container resizing. Contributed by Meng Ding

2015-07-28 Thread jianhe
YARN-3867. ContainerImpl changes to support container resizing. Contributed by 
Meng Ding


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb956626
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb956626
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb956626

Branch: refs/heads/YARN-1197
Commit: cb956626955a60a9d972b4376bc1b334d3b14993
Parents: 4facdce
Author: Jian He 
Authored: Tue Jul 28 13:51:23 2015 -0700
Committer: Jian He 
Committed: Tue Jul 28 13:53:33 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |  3 ++
 .../hadoop/yarn/server/utils/BuilderUtils.java  |  4 ++-
 .../containermanager/ContainerManagerImpl.java  |  7 ++--
 .../container/ChangeContainerResourceEvent.java | 36 ---
 .../containermanager/container/Container.java   |  2 ++
 .../container/ContainerEventType.java   |  4 ---
 .../container/ContainerImpl.java| 16 ++---
 .../ChangeMonitoringContainerResourceEvent.java | 37 
 .../monitor/ContainersMonitorEventType.java |  3 +-
 .../nodemanager/metrics/NodeManagerMetrics.java | 11 ++
 .../nodemanager/TestNodeStatusUpdater.java  |  2 +-
 .../metrics/TestNodeManagerMetrics.java | 18 +++---
 .../nodemanager/webapp/MockContainer.java   |  4 +++
 .../yarn/server/resourcemanager/MockNM.java |  2 +-
 .../server/resourcemanager/NodeManager.java |  2 +-
 .../resourcemanager/TestApplicationCleanup.java |  6 ++--
 .../attempt/TestRMAppAttemptTransitions.java| 21 +++
 .../capacity/TestCapacityScheduler.java |  2 +-
 .../scheduler/fifo/TestFifoScheduler.java   |  4 +--
 .../security/TestAMRMTokens.java|  3 +-
 20 files changed, 118 insertions(+), 69 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb956626/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index eace759..40f0be3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -165,6 +165,9 @@ Release 2.8.0 - UNRELEASED
 YARN-1645. ContainerManager implementation to support container resizing.
 (Meng Ding & Wangda Tan via jianhe)
 
+YARN-3867. ContainerImpl changes to support container resizing. (Meng Ding 
+via jianhe)
+
   IMPROVEMENTS
 
 YARN-644. Basic null check is not performed on passed in arguments before

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb956626/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index f2146c8..5f92c0c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -200,13 +200,15 @@ public class BuilderUtils {
   }
 
   public static ContainerStatus newContainerStatus(ContainerId containerId,
-  ContainerState containerState, String diagnostics, int exitStatus) {
+  ContainerState containerState, String diagnostics, int exitStatus,
+  Resource capability) {
 ContainerStatus containerStatus = recordFactory
   .newRecordInstance(ContainerStatus.class);
 containerStatus.setState(containerState);
 containerStatus.setContainerId(containerId);
 containerStatus.setDiagnostics(diagnostics);
 containerStatus.setExitStatus(exitStatus);
+containerStatus.setCapability(capability);
 return containerStatus;
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb956626/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.ja

[02/50] [abbrv] hadoop git commit: YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo should based on total-used-resources. (Bibin A Chundatt via wangda)

2015-07-28 Thread jianhe
YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo 
should based on total-used-resources. (Bibin A Chundatt via wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76ec26de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76ec26de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76ec26de

Branch: refs/heads/YARN-1197
Commit: 76ec26de8099dc48ce3812c595b7ab857a600442
Parents: 1b3bceb
Author: Wangda Tan 
Authored: Wed Jul 22 11:54:02 2015 -0700
Committer: Wangda Tan 
Committed: Wed Jul 22 11:54:02 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |  3 +
 .../scheduler/SchedulerApplicationAttempt.java  |  2 +-
 .../scheduler/capacity/LeafQueue.java   |  8 ++-
 .../TestCapacitySchedulerNodeLabelUpdate.java   | 64 
 4 files changed, 74 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5100cdf..f751862 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -653,6 +653,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3885. ProportionalCapacityPreemptionPolicy doesn't preempt if queue 
is 
 more than 2 level. (Ajith S via wangda)
 
+YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo 
+should based on total-used-resources. (Bibin A Chundatt via wangda)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index cf543bd..317e61c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -598,7 +598,7 @@ public class SchedulerApplicationAttempt implements 
SchedulableEntity {
 AggregateAppResourceUsage runningResourceUsage =
 getRunningAggregateAppResourceUsage();
 Resource usedResourceClone =
-Resources.clone(attemptResourceUsage.getUsed());
+Resources.clone(attemptResourceUsage.getAllUsed());
 Resource reservedResourceClone =
 Resources.clone(attemptResourceUsage.getReserved());
 return ApplicationResourceUsageReport.newInstance(liveContainers.size(),

http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 0ce4d68..5c283f4 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -439,7 +439,7 @@ public class LeafQueue extends AbstractCSQueue {
 for (Map.Entry entry : users.entrySet()) {
   User user = entry.getValue();
   usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(user
-  .getUsed()), user.getActiveApplications(), user
+  .getAllUsed()), user.getActiveApplications(), user
   .getPendingApplications(), Resources.clone(user
   .getConsumedA

[35/50] [abbrv] hadoop git commit: YARN-3958. TestYarnConfigurationFields should be moved to hadoop-yarn-api module. Contributed by Varun Saxena.

2015-07-28 Thread jianhe
YARN-3958. TestYarnConfigurationFields should be moved to hadoop-yarn-api 
module. Contributed by Varun Saxena.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42d4e0ae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42d4e0ae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42d4e0ae

Branch: refs/heads/YARN-1197
Commit: 42d4e0ae99d162fde52902cb86e29f2c82a084c8
Parents: 156f24e
Author: Akira Ajisaka 
Authored: Mon Jul 27 11:43:25 2015 +0900
Committer: Akira Ajisaka 
Committed: Mon Jul 27 11:43:25 2015 +0900

--
 hadoop-yarn-project/CHANGES.txt |   3 +
 .../hadoop-yarn/hadoop-yarn-api/pom.xml |  34 +
 .../yarn/conf/TestYarnConfigurationFields.java  | 136 +++
 .../yarn/conf/TestYarnConfigurationFields.java  | 136 ---
 4 files changed, 173 insertions(+), 136 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/42d4e0ae/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 883d009..3b7d8a8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -685,6 +685,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3973. Recent changes to application priority management break 
 reservation system from YARN-1051. (Carlo Curino via wangda)
 
+YARN-3958. TestYarnConfigurationFields should be moved to hadoop-yarn-api
+module. (Varun Saxena via aajisaka)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/42d4e0ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
index dc9c469..5c4156b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
@@ -62,9 +62,31 @@
   com.google.protobuf
   protobuf-java
 
+
+
+  org.apache.hadoop
+  hadoop-common
+  test-jar
+  test
+
+
+
+  junit
+  junit
+  test
+
   
 
   
+
+  
+
${basedir}/../hadoop-yarn-common/src/main/resources
+
+  yarn-default.xml
+
+false
+  
+
 
   
 org.apache.hadoop
@@ -105,6 +127,18 @@
   
 
   
+
+  
+maven-jar-plugin
+
+  
+
+  test-jar
+
+test-compile
+  
+
+  
 
   
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/42d4e0ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
new file mode 100644
index 000..e89a90d
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.conf;
+
+import java.util.HashSet;
+
+import org.apache.hadoop.conf.TestConfigurationFieldsBase;
+
+/**
+ * Unit test class to compare
+ * {@link org.apache.hadoop.yarn.conf.YarnConfiguration} and
+ * yarn-default.xml for missing properties.  Currently only throws an error
+ * if the class is missing a property.
+ * 
+ * Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase}
+ * for how this class works.
+ */
+public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
+
+  @Suppr

[28/50] [abbrv] hadoop git commit: HADOOP-12135. cleanup releasedocmaker

2015-07-28 Thread jianhe
HADOOP-12135. cleanup releasedocmaker

(cherry picked from commit 3fee9f8d18dd60d83da674b3cfbefe666915fad8)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8b62d11
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8b62d11
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8b62d11

Branch: refs/heads/YARN-1197
Commit: e8b62d11d460e9706e48df92a0b0a72f4a02d3f5
Parents: 098ba45
Author: Allen Wittenauer 
Authored: Mon Jul 6 15:49:03 2015 -0700
Committer: Andrew Wang 
Committed: Fri Jul 24 18:31:30 2015 -0700

--
 dev-support/releasedocmaker.py | 384 +++-
 1 file changed, 207 insertions(+), 177 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b62d11/dev-support/releasedocmaker.py
--
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index 8e68b3c..6e01260 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -19,6 +19,7 @@
 from glob import glob
 from optparse import OptionParser
 from time import gmtime, strftime
+import pprint
 import os
 import re
 import sys
@@ -99,23 +100,44 @@ def mstr(obj):
 return ""
   return unicode(obj)
 
-def buildindex(master):
+def buildindex(title,license):
   versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
   with open("index.md","w") as indexfile:
+if license is True:
+  indexfile.write(asflicense)
 for v in versions:
-  indexfile.write("* Apache Hadoop v%s\n" % (v))
+  indexfile.write("* %s v%s\n" % (title,v))
   for k in ("Changes","Release Notes"):
-indexfile.write("*  %s\n" %(k))
-indexfile.write("* [Combined %s](%s/%s.%s.html)\n" \
+indexfile.write("* %s (%s/%s.%s.html)\n" \
   % (k,v,k.upper().replace(" ",""),v))
-if not master:
-  indexfile.write("* [Hadoop Common 
%s](%s/%s.HADOOP.%s.html)\n" \
-% (k,v,k.upper().replace(" ",""),v))
-  for p in ("HDFS","MapReduce","YARN"):
-indexfile.write("* [%s %s](%s/%s.%s.%s.html)\n" \
-  % (p,k,v,k.upper().replace(" ",""),p.upper(),v))
   indexfile.close()
 
+class GetVersions:
+  """ yo """
+  def __init__(self,versions, projects):
+versions = versions
+projects = projects
+self.newversions = []
+pp = pprint.PrettyPrinter(indent=4)
+at=0
+end=1
+count=100
+versions.sort()
+print "Looking for %s through %s"%(versions[0],versions[-1])
+for p in projects:
+  resp = 
urllib.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
+  data = json.loads(resp.read())
+  for d in data:
+if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= 
versions[-1]:
+  print "Adding %s to the list" % d['name']
+  self.newversions.append(d['name'])
+newlist=list(set(self.newversions))
+self.newversions=newlist
+
+  def getlist(self):
+  pp = pprint.PrettyPrinter(indent=4)
+  return(self.newversions)
+
 class Version:
   """Represents a version number"""
   def __init__(self, data):
@@ -261,8 +283,10 @@ class Jira:
 class JiraIter:
   """An Iterator of JIRAs"""
 
-  def __init__(self, versions):
-self.versions = versions
+  def __init__(self, version, projects):
+self.version = version
+self.projects = projects
+v=str(version).replace("-SNAPSHOT","")
 
 resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field";)
 data = json.loads(resp.read())
@@ -276,7 +300,7 @@ class JiraIter:
 end=1
 count=100
 while (at < end):
-  params = urllib.urlencode({'jql': "project in 
(HADOOP,HDFS,MAPREDUCE,YARN) and fixVersion in ('"+"' , 
'".join([str(v).replace("-SNAPSHOT","") for v in versions])+"') and resolution 
= Fixed", 'startAt':at, 'maxResults':count})
+  params = urllib.urlencode({'jql': "project in ('"+"' , 
'".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 
'startAt':at, 'maxResults':count})
   resp = 
urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
   data = json.loads(resp.read())
   if (data.has_key('errorMessages')):
@@ -286,10 +310,8 @@ class JiraIter:
   self.jiras.extend(data['issues'])
 
   needaversion=False
-  for j in versions:
-v=str(j).replace("-SNAPSHOT","")
-if v not in releaseVersion:
-  needaversion=True
+  if v not in releaseVersion:
+needaversion=True
 
   if needaversion is True:
 for i in range(len(data['issues'])):
@@ -351,21 +373,29 @@ class Outputs:
   self.writeKeyRaw(jira.getProject(), line)
 
 def main():
-  parser = OptionParser(usage="usage: %prog --version VERSION [--ver

[24/50] [abbrv] hadoop git commit: YARN-3026. Move application-specific container allocation logic from LeafQueue to FiCaSchedulerApp. Contributed by Wangda Tan

2015-07-28 Thread jianhe
YARN-3026. Move application-specific container allocation logic from LeafQueue 
to FiCaSchedulerApp. Contributed by Wangda Tan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83fe34ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83fe34ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83fe34ac

Branch: refs/heads/YARN-1197
Commit: 83fe34ac0896cee0918bbfad7bd51231e4aec39b
Parents: fc42fa8
Author: Jian He 
Authored: Fri Jul 24 14:00:25 2015 -0700
Committer: Jian He 
Committed: Fri Jul 24 14:00:25 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |   3 +
 .../server/resourcemanager/RMContextImpl.java   |   3 +-
 .../scheduler/ResourceLimits.java   |  19 +-
 .../scheduler/capacity/AbstractCSQueue.java |  27 +-
 .../scheduler/capacity/CSAssignment.java|  12 +-
 .../capacity/CapacityHeadroomProvider.java  |  16 +-
 .../scheduler/capacity/CapacityScheduler.java   |  14 -
 .../scheduler/capacity/LeafQueue.java   | 833 +++
 .../scheduler/capacity/ParentQueue.java |  16 +-
 .../scheduler/common/fica/FiCaSchedulerApp.java | 721 +++-
 .../capacity/TestApplicationLimits.java |  15 +-
 .../capacity/TestCapacityScheduler.java |   3 +-
 .../capacity/TestContainerAllocation.java   |  85 +-
 .../scheduler/capacity/TestLeafQueue.java   | 191 +
 .../scheduler/capacity/TestReservations.java| 111 +--
 .../scheduler/capacity/TestUtils.java   |  25 +-
 16 files changed, 1048 insertions(+), 1046 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/83fe34ac/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d1546b2..cf00fe5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -345,6 +345,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3844. Make hadoop-yarn-project Native code -Wall-clean (Alan Burlison
 via Colin P. McCabe)
 
+YARN-3026. Move application-specific container allocation logic from
+LeafQueue to FiCaSchedulerApp. (Wangda Tan via jianhe)
+
   OPTIMIZATIONS
 
 YARN-3339. TestDockerContainerExecutor should pull a single image and not

http://git-wip-us.apache.org/repos/asf/hadoop/blob/83fe34ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 2f9209c..8cadc3b 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -292,7 +292,8 @@ public class RMContextImpl implements RMContext {
 activeServiceContext.setNMTokenSecretManager(nmTokenSecretManager);
   }
 
-  void setScheduler(ResourceScheduler scheduler) {
+  @VisibleForTesting
+  public void setScheduler(ResourceScheduler scheduler) {
 activeServiceContext.setScheduler(scheduler);
   }
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/83fe34ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
index 8074794..c545e9e 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
@@ -26,20 +26,25 @@ import org.apache.hadoop.yarn.util.res

[42/50] [abbrv] hadoop git commit: HADOOP-12245. References to misspelled REMAINING_QUATA in FileSystemShell.md. Contributed by Gabor Liptak.

2015-07-28 Thread jianhe
HADOOP-12245. References to misspelled REMAINING_QUATA in FileSystemShell.md. 
Contributed by Gabor Liptak.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e21dde50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e21dde50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e21dde50

Branch: refs/heads/YARN-1197
Commit: e21dde501aa9323b7f34b4bc4ba9d282ec4f2707
Parents: 3572ebd
Author: Akira Ajisaka 
Authored: Tue Jul 28 11:33:10 2015 +0900
Committer: Akira Ajisaka 
Committed: Tue Jul 28 11:33:10 2015 +0900

--
 hadoop-common-project/hadoop-common/CHANGES.txt   | 3 +++
 .../hadoop-common/src/site/markdown/FileSystemShell.md| 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e21dde50/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index baf39e3..aeaa5b9 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1017,6 +1017,9 @@ Release 2.8.0 - UNRELEASED
 HADOOP-12239. StorageException complaining " no lease ID" when updating
 FolderLastModifiedTime in WASB. (Duo Xu via cnauroth)
 
+HADOOP-12245. References to misspelled REMAINING_QUATA in
+FileSystemShell.md. (Gabor Liptak via aajisaka)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e21dde50/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
--
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 144cb73..fb89ca1 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -174,7 +174,7 @@ Usage: `hadoop fs -count [-q] [-h] [-v]  `
 
 Count the number of directories, files and bytes under the paths that match 
the specified file pattern. The output columns with -count are: DIR\_COUNT, 
FILE\_COUNT, CONTENT\_SIZE, PATHNAME
 
-The output columns with -count -q are: QUOTA, REMAINING\_QUATA, SPACE\_QUOTA, 
REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
+The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, 
REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
 
 The -h option shows sizes in human readable format.
 



[46/50] [abbrv] hadoop git commit: YARN-1449. AM-NM protocol changes to support container resizing. Contributed by Meng Ding & Wangda Tan)

2015-07-28 Thread jianhe
YARN-1449. AM-NM protocol changes to support container resizing. Contributed by 
Meng Ding & Wangda Tan)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3f2830d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3f2830d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3f2830d

Branch: refs/heads/YARN-1197
Commit: b3f2830df4b3b0a35d0b2d5542cae9dfecb2b397
Parents: 7758afd
Author: Jian He 
Authored: Tue Jul 14 16:06:25 2015 -0700
Committer: Jian He 
Committed: Tue Jul 28 13:53:32 2015 -0700

--
 .../v2/app/launcher/TestContainerLauncher.java  |  11 +
 .../app/launcher/TestContainerLauncherImpl.java |   9 +
 hadoop-yarn-project/CHANGES.txt |   3 +
 .../yarn/api/ContainerManagementProtocol.java   |  30 ++-
 .../IncreaseContainersResourceRequest.java  |  75 ++
 .../IncreaseContainersResourceResponse.java |  93 +++
 .../proto/containermanagement_protocol.proto|   1 +
 ...ContainerManagementProtocolPBClientImpl.java |  20 ++
 ...ontainerManagementProtocolPBServiceImpl.java |  22 ++
 ...IncreaseContainersResourceRequestPBImpl.java | 170 +
 ...ncreaseContainersResourceResponsePBImpl.java | 241 +++
 .../hadoop/yarn/TestContainerLaunchRPC.java |   8 +
 .../yarn/TestContainerResourceIncreaseRPC.java  | 162 +
 .../java/org/apache/hadoop/yarn/TestRPC.java|   8 +
 .../hadoop/yarn/api/TestPBImplRecords.java  |  20 ++
 .../containermanager/ContainerManagerImpl.java  |  13 +
 .../server/resourcemanager/NodeManager.java |  11 +-
 .../resourcemanager/TestAMAuthorization.java|   8 +
 .../TestApplicationMasterLauncher.java  |   9 +
 19 files changed, 910 insertions(+), 4 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3f2830d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
index 41ee65d..6c3a4d6 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
@@ -30,6 +30,8 @@ import java.util.Map;
 import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.atomic.AtomicInteger;
 
+import 
org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import 
org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
 import org.junit.Assert;
 
 import org.apache.commons.logging.Log;
@@ -449,5 +451,14 @@ public class TestContainerLauncher {
   "Dummy function cause"));
   throw new IOException(e);
 }
+
+@Override
+public IncreaseContainersResourceResponse increaseContainersResource(
+IncreaseContainersResourceRequest request) throws IOException,
+IOException {
+  Exception e = new Exception("Dummy function", new Exception(
+  "Dummy function cause"));
+  throw new IOException(e);
+}
   }
 }

http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3f2830d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
--
diff --git 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
index 184f1b2..610448c 100644
--- 
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
+++ 
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
@@ -46,6 +46,8 @@ import 
org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
 import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher.EventType;
 import org.a

[30/50] [abbrv] hadoop git commit: HADOOP-12237. releasedocmaker.py doesn't work behind a proxy (Tsuyoshi Ozawa via aw)

2015-07-28 Thread jianhe
HADOOP-12237. releasedocmaker.py doesn't work behind a proxy (Tsuyoshi Ozawa 
via aw)

(cherry picked from commit b41fe3111ae37478cbace2a07e6ac35a676ef978)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adcf5dd9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adcf5dd9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adcf5dd9

Branch: refs/heads/YARN-1197
Commit: adcf5dd94052481f66deaf402ac4ace1ffc06f49
Parents: d769783
Author: Allen Wittenauer 
Authored: Mon Jul 20 09:47:46 2015 -0700
Committer: Andrew Wang 
Committed: Fri Jul 24 18:31:48 2015 -0700

--
 dev-support/releasedocmaker.py | 11 ---
 1 file changed, 8 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/adcf5dd9/dev-support/releasedocmaker.py
--
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index 409d8e3..d2e5dda 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -24,6 +24,7 @@ import os
 import re
 import sys
 import urllib
+import urllib2
 try:
   import json
 except ImportError:
@@ -125,7 +126,7 @@ class GetVersions:
 versions.sort()
 print "Looking for %s through %s"%(versions[0],versions[-1])
 for p in projects:
-  resp = 
urllib.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
+  resp = 
urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
   data = json.loads(resp.read())
   for d in data:
 if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <= 
versions[-1]:
@@ -288,7 +289,7 @@ class JiraIter:
 self.projects = projects
 v=str(version).replace("-SNAPSHOT","")
 
-resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field";)
+resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/field";)
 data = json.loads(resp.read())
 
 self.fieldIdMap = {}
@@ -301,7 +302,7 @@ class JiraIter:
 count=100
 while (at < end):
   params = urllib.urlencode({'jql': "project in ('"+"' , 
'".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed", 
'startAt':at, 'maxResults':count})
-  resp = 
urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
+  resp = 
urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
   data = json.loads(resp.read())
   if (data.has_key('errorMessages')):
 raise Exception(data['errorMessages'])
@@ -407,6 +408,10 @@ def main():
   if (len(options.versions) <= 0):
 parser.error("At least one version needs to be supplied")
 
+  proxy = urllib2.ProxyHandler()
+  opener = urllib2.build_opener(proxy)
+  urllib2.install_opener(opener)
+
   projects = options.projects
 
   if (options.range is True):



[36/50] [abbrv] hadoop git commit: HDFS-8810. Correct assertions in TestDFSInotifyEventInputStream class. Contributed by Surendra Singh Lilhore.

2015-07-28 Thread jianhe
HDFS-8810. Correct assertions in TestDFSInotifyEventInputStream class. 
Contributed by Surendra Singh Lilhore.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1df78688
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1df78688
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1df78688

Branch: refs/heads/YARN-1197
Commit: 1df78688c69476f89d16f93bc74a4f05d0b1a3da
Parents: 42d4e0a
Author: Akira Ajisaka 
Authored: Mon Jul 27 13:17:24 2015 +0900
Committer: Akira Ajisaka 
Committed: Mon Jul 27 13:17:24 2015 +0900

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt  | 3 +++
 .../org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java   | 4 ++--
 2 files changed, 5 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1df78688/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3614e01..1ddf7da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1081,6 +1081,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-8773. Few FSNamesystem metrics are not documented in the Metrics page.
 (Rakesh R via cnauroth)
 
+HDFS-8810. Correct assertions in TestDFSInotifyEventInputStream class.
+(Surendra Singh Lilhore via aajisaka)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1df78688/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
index 65569d0..e7bbcac 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
@@ -164,7 +164,7 @@ public class TestDFSInotifyEventInputStream {
   Event.RenameEvent re2 = (Event.RenameEvent) batch.getEvents()[0];
   Assert.assertTrue(re2.getDstPath().equals("/file2"));
   Assert.assertTrue(re2.getSrcPath().equals("/file4"));
-  Assert.assertTrue(re.getTimestamp() > 0);
+  Assert.assertTrue(re2.getTimestamp() > 0);
   LOG.info(re2.toString());
 
   // AddOp with overwrite
@@ -378,7 +378,7 @@ public class TestDFSInotifyEventInputStream {
   Event.RenameEvent re3 = (Event.RenameEvent) batch.getEvents()[0];
   Assert.assertTrue(re3.getDstPath().equals("/dir/file5"));
   Assert.assertTrue(re3.getSrcPath().equals("/file5"));
-  Assert.assertTrue(re.getTimestamp() > 0);
+  Assert.assertTrue(re3.getTimestamp() > 0);
   LOG.info(re3.toString());
 
   // TruncateOp



[06/50] [abbrv] hadoop git commit: YARN-2019. Retrospect on decision of making RM crashed if any exception throw in ZKRMStateStore. Contributed by Jian He.

2015-07-28 Thread jianhe
YARN-2019. Retrospect on decision of making RM crashed if any exception throw 
in ZKRMStateStore. Contributed by Jian He.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee98d635
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee98d635
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee98d635

Branch: refs/heads/YARN-1197
Commit: ee98d6354bbbcd0832d3e539ee097f837e5d0e31
Parents: e91ccfa
Author: Junping Du 
Authored: Wed Jul 22 17:52:35 2015 -0700
Committer: Junping Du 
Committed: Wed Jul 22 17:52:35 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |  3 +++
 .../apache/hadoop/yarn/conf/YarnConfiguration.java  | 11 +++
 .../src/main/resources/yarn-default.xml | 16 
 .../resourcemanager/recovery/RMStateStore.java  |  9 +++--
 4 files changed, 37 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee98d635/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a5fd4e7..93962f1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -144,6 +144,9 @@ Release 2.8.0 - UNRELEASED
 YARN-2003. Support for Application priority : Changes in RM and Capacity 
 Scheduler. (Sunil G via wangda)
 
+YARN-2019. Retrospect on decision of making RM crashed if any exception 
throw 
+in ZKRMStateStore. (Jian He via junping_du)
+
   IMPROVEMENTS
 
 YARN-644. Basic null check is not performed on passed in arguments before

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee98d635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 060635f..9832729 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -401,6 +401,11 @@ public class YarnConfiguration extends Configuration {
   public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled";
   public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false;
 
+  public static final String YARN_FAIL_FAST = YARN_PREFIX + "fail-fast";
+  public static final boolean DEFAULT_YARN_FAIL_FAST = true;
+
+  public static final String RM_FAIL_FAST = RM_PREFIX + "fail-fast";
+
   @Private
   public static final String RM_WORK_PRESERVING_RECOVERY_ENABLED = RM_PREFIX
   + "work-preserving-recovery.enabled";
@@ -2018,6 +2023,12 @@ public class YarnConfiguration extends Configuration {
 YARN_HTTP_POLICY_DEFAULT));
   }
 
+  public static boolean shouldRMFailFast(Configuration conf) {
+return conf.getBoolean(YarnConfiguration.RM_FAIL_FAST,
+conf.getBoolean(YarnConfiguration.YARN_FAIL_FAST,
+YarnConfiguration.DEFAULT_YARN_FAIL_FAST));
+  }
+
   @Private
   public static String getClusterId(Configuration conf) {
 String clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID);

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee98d635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index d586f51..8b3a3af 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -324,6 +324,22 @@
   
 
   
+Should RM fail fast if it encounters any errors. By defalt, it
+  points to ${yarn.fail-fast}. Errors include:
+  1) exceptions when state-store write/read operations fails.
+
+yarn.resourcemanager.fail-fast
+${yarn.fail-fast}
+  
+
+  
+Should YARN fail fast if it encounters any errors.
+
+yarn.fail-fast
+true
+  
+
+  
 Enable RM work preserving recovery. This configuration is 
private
 to YARN for experimenting the feature.
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee98d635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/se

[10/50] [abbrv] hadoop git commit: HADOOP-12189. Improve CallQueueManager#swapQueue to make queue elements drop nearly impossible. Contributed by Zhihai Xu.

2015-07-28 Thread jianhe
HADOOP-12189. Improve CallQueueManager#swapQueue to make queue elements drop 
nearly impossible. Contributed by Zhihai Xu.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6736a1ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6736a1ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6736a1ab

Branch: refs/heads/YARN-1197
Commit: 6736a1ab7033523ed5f304fdfed46d7f348665b4
Parents: 813cf89
Author: Andrew Wang 
Authored: Thu Jul 23 14:42:35 2015 -0700
Committer: Andrew Wang 
Committed: Thu Jul 23 14:42:35 2015 -0700

--
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../org/apache/hadoop/ipc/CallQueueManager.java | 27 +---
 .../apache/hadoop/ipc/TestCallQueueManager.java |  6 ++---
 3 files changed, 24 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/6736a1ab/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index f1a3bc9..6c18add 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -719,6 +719,9 @@ Release 2.8.0 - UNRELEASED
 HADOOP-12161. Add getStoragePolicy API to the FileSystem interface.
 (Brahma Reddy Battula via Arpit Agarwal)
 
+HADOOP-12189. Improve CallQueueManager#swapQueue to make queue elements
+drop nearly impossible. (Zhihai Xu via wang)
+
   OPTIMIZATIONS
 
 HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6736a1ab/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
index 1568bd6..c10f839 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
@@ -32,11 +32,15 @@ import org.apache.hadoop.conf.Configuration;
  */
 public class CallQueueManager {
   public static final Log LOG = LogFactory.getLog(CallQueueManager.class);
+  // Number of checkpoints for empty queue.
+  private static final int CHECKPOINT_NUM = 20;
+  // Interval to check empty queue.
+  private static final long CHECKPOINT_INTERVAL_MS = 10;
 
   @SuppressWarnings("unchecked")
   static  Class> convertQueueClass(
-  Class queneClass, Class elementClass) {
-return (Class>)queneClass;
+  Class queueClass, Class elementClass) {
+return (Class>)queueClass;
   }
   private final boolean clientBackOffEnabled;
 
@@ -159,18 +163,23 @@ public class CallQueueManager {
   }
 
   /**
-   * Checks if queue is empty by checking at two points in time.
+   * Checks if queue is empty by checking at CHECKPOINT_NUM points with
+   * CHECKPOINT_INTERVAL_MS interval.
* This doesn't mean the queue might not fill up at some point later, but
* it should decrease the probability that we lose a call this way.
*/
   private boolean queueIsReallyEmpty(BlockingQueue q) {
-boolean wasEmpty = q.isEmpty();
-try {
-  Thread.sleep(10);
-} catch (InterruptedException ie) {
-  return false;
+for (int i = 0; i < CHECKPOINT_NUM; i++) {
+  try {
+Thread.sleep(CHECKPOINT_INTERVAL_MS);
+  } catch (InterruptedException ie) {
+return false;
+  }
+  if (!q.isEmpty()) {
+return false;
+  }
 }
-return q.isEmpty() && wasEmpty;
+return true;
   }
 
   private String stringRepr(Object o) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/6736a1ab/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
index 6e1838e..51a9750 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
@@ -165,7 +165,7 @@ public class TestCallQueueManager {
 HashMap threads = new HashMap();
 
 // Create putters and takers
-for (int i=0; i < 50; i++) {
+for (int i=0; i < 1000; i++) {
   Putter p = new Putter(manager

[21/50] [abbrv] hadoop git commit: HADOOP-12170. hadoop-common's JNIFlags.cmake is redundant and can be removed (Alan Burlison via Colin P. McCabe)

2015-07-28 Thread jianhe
HADOOP-12170. hadoop-common's JNIFlags.cmake is redundant and can be removed 
(Alan Burlison via Colin P. McCabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4b0c744
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4b0c744
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4b0c744

Branch: refs/heads/YARN-1197
Commit: e4b0c74434b82c25256a59b03d62b1a66bb8ac69
Parents: d19d187
Author: Colin Patrick Mccabe 
Authored: Fri Jul 24 13:03:31 2015 -0700
Committer: Colin Patrick Mccabe 
Committed: Fri Jul 24 13:03:31 2015 -0700

--
 hadoop-common-project/hadoop-common/CHANGES.txt |   3 +
 .../hadoop-common/src/JNIFlags.cmake| 124 ---
 2 files changed, 3 insertions(+), 124 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4b0c744/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index d6d43f2..0da6194 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -727,6 +727,9 @@ Release 2.8.0 - UNRELEASED
 
 HADOOP-12259. Utility to Dynamic port allocation (brahmareddy via rkanter)
 
+HADOOP-12170. hadoop-common's JNIFlags.cmake is redundant and can be
+removed (Alan Burlison via Colin P. McCabe)
+
   OPTIMIZATIONS
 
 HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4b0c744/hadoop-common-project/hadoop-common/src/JNIFlags.cmake
--
diff --git a/hadoop-common-project/hadoop-common/src/JNIFlags.cmake 
b/hadoop-common-project/hadoop-common/src/JNIFlags.cmake
deleted file mode 100644
index c558fe8..000
--- a/hadoop-common-project/hadoop-common/src/JNIFlags.cmake
+++ /dev/null
@@ -1,124 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-
-# If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit.
-# This variable is set by maven.
-if (JVM_ARCH_DATA_MODEL EQUAL 32)
-# Force 32-bit code generation on amd64/x86_64, ppc64, sparc64
-if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_SYSTEM_PROCESSOR MATCHES ".*64")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")
-set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32")
-endif ()
-if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR 
STREQUAL "amd64")
-# Set CMAKE_SYSTEM_PROCESSOR to ensure that find_package(JNI) will use
-# the 32-bit version of libjvm.so.
-set(CMAKE_SYSTEM_PROCESSOR "i686")
-endif ()
-endif (JVM_ARCH_DATA_MODEL EQUAL 32)
-
-# Determine float ABI of JVM on ARM Linux
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm" AND CMAKE_SYSTEM_NAME STREQUAL 
"Linux")
-find_program(READELF readelf)
-if (READELF MATCHES "NOTFOUND")
-message(WARNING "readelf not found; JVM float ABI detection disabled")
-else (READELF MATCHES "NOTFOUND")
-execute_process(
-COMMAND ${READELF} -A ${JAVA_JVM_LIBRARY}
-OUTPUT_VARIABLE JVM_ELF_ARCH
-ERROR_QUIET)
-if (NOT JVM_ELF_ARCH MATCHES "Tag_ABI_VFP_args: VFP registers")
-message("Soft-float JVM detected")
-
-# Test compilation with -mfloat-abi=softfp using an arbitrary libc 
function
-# (typically fails with "fatal error: bits/predefs.h: No such file 
or directory"
-# if soft-float dev libraries are not installed)
-include(CMakePushCheckState)
-cmake_push_check_state()
-set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} 
-mfloat-abi=softfp")
-include(CheckSymbolExists)
-check_symbol_exists(exit stdlib.h SOFTFP_AVAILABLE)
-if (NOT SOFTFP_AVAILABLE)
-message(FATAL_ERROR "Soft-flo

[13/50] [abbrv] hadoop git commit: YARN-3969. Allow jobs to be submitted to reservation that is active but does not have any allocations. (subru via curino)

2015-07-28 Thread jianhe
YARN-3969. Allow jobs to be submitted to reservation that is active but does 
not have any allocations. (subru via curino)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0fcb4a8c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0fcb4a8c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0fcb4a8c

Branch: refs/heads/YARN-1197
Commit: 0fcb4a8cf2add3f112907ff4e833e2f04947b53e
Parents: 206d493
Author: carlo curino 
Authored: Thu Jul 23 19:33:59 2015 -0700
Committer: carlo curino 
Committed: Thu Jul 23 19:33:59 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |  3 +++
 .../scheduler/capacity/ReservationQueue.java|  4 ---
 .../capacity/TestReservationQueue.java  | 26 +++-
 3 files changed, 17 insertions(+), 16 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fcb4a8c/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f23853b..8bc9e4c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -864,6 +864,9 @@ Release 2.7.1 - 2015-07-06
 YARN-3850. NM fails to read files from full disks which can lead to
 container logs being lost and other issues (Varun Saxena via jlowe)
 
+YARN-3969. Allow jobs to be submitted to reservation that is active 
+but does not have any allocations. (subru via curino)
+
 Release 2.7.0 - 2015-04-20
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fcb4a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
index 4790cc7..976cf8c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
@@ -39,12 +39,9 @@ public class ReservationQueue extends LeafQueue {
 
   private PlanQueue parent;
 
-  private int maxSystemApps;
-
   public ReservationQueue(CapacitySchedulerContext cs, String queueName,
   PlanQueue parent) throws IOException {
 super(cs, queueName, parent, null);
-maxSystemApps = cs.getConfiguration().getMaximumSystemApplications();
 // the following parameters are common to all reservation in the plan
 updateQuotas(parent.getUserLimitForReservation(),
 parent.getUserLimitFactor(),
@@ -89,7 +86,6 @@ public class ReservationQueue extends LeafQueue {
 }
 setCapacity(capacity);
 setAbsoluteCapacity(getParent().getAbsoluteCapacity() * getCapacity());
-setMaxApplications((int) (maxSystemApps * getAbsoluteCapacity()));
 // note: we currently set maxCapacity to capacity
 // this might be revised later
 setMaxCapacity(entitlement.getMaxCapacity());

http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fcb4a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
index 4e6c73d..e23e93c 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.serv

[19/50] [abbrv] hadoop git commit: HDFS-8735. Inotify: All events classes should implement toString() API. Contributed by Surendra Singh Lilhore.

2015-07-28 Thread jianhe
HDFS-8735. Inotify: All events classes should implement toString() API. 
Contributed by Surendra Singh Lilhore.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8f60918
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8f60918
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8f60918

Branch: refs/heads/YARN-1197
Commit: f8f60918230dd466ae8dda1fbc28878e19273232
Parents: fbd6063
Author: Akira Ajisaka 
Authored: Sat Jul 25 02:56:55 2015 +0900
Committer: Akira Ajisaka 
Committed: Sat Jul 25 02:56:55 2015 +0900

--
 .../org/apache/hadoop/hdfs/inotify/Event.java   | 95 
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 +
 .../hdfs/TestDFSInotifyEventInputStream.java| 26 ++
 3 files changed, 124 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8f60918/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
index dee17a9..6f2b5e2 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
@@ -51,6 +51,7 @@ public abstract class Event {
   /**
* Sent when a file is closed after append or create.
*/
+  @InterfaceAudience.Public
   public static class CloseEvent extends Event {
 private String path;
 private long fileSize;
@@ -81,11 +82,20 @@ public abstract class Event {
 public long getTimestamp() {
   return timestamp;
 }
+
+@Override
+@InterfaceStability.Unstable
+public String toString() {
+  return "CloseEvent [path=" + path + ", fileSize=" + fileSize
+  + ", timestamp=" + timestamp + "]";
+}
+
   }
 
   /**
* Sent when a new file is created (including overwrite).
*/
+  @InterfaceAudience.Public
   public static class CreateEvent extends Event {
 
 public static enum INodeType {
@@ -232,6 +242,25 @@ public abstract class Event {
 public long getDefaultBlockSize() {
   return defaultBlockSize;
 }
+
+@Override
+@InterfaceStability.Unstable
+public String toString() {
+  StringBuilder content = new StringBuilder();
+  content.append("CreateEvent [INodeType=" + iNodeType + ", path=" + path
+  + ", ctime=" + ctime + ", replication=" + replication
+  + ", ownerName=" + ownerName + ", groupName=" + groupName
+  + ", perms=" + perms + ", ");
+
+  if (symlinkTarget != null) {
+content.append("symlinkTarget=" + symlinkTarget + ", ");
+  }
+
+  content.append("overwrite=" + overwrite + ", defaultBlockSize="
+  + defaultBlockSize + "]");
+  return content.toString();
+}
+
   }
 
   /**
@@ -242,6 +271,7 @@ public abstract class Event {
* metadataType of the MetadataUpdateEvent will be null or will have their 
default
* values.
*/
+  @InterfaceAudience.Public
   public static class MetadataUpdateEvent extends Event {
 
 public static enum MetadataType {
@@ -400,11 +430,45 @@ public abstract class Event {
   return xAttrsRemoved;
 }
 
+@Override
+@InterfaceStability.Unstable
+public String toString() {
+  StringBuilder content = new StringBuilder();
+  content.append("MetadataUpdateEvent [path=" + path + ", metadataType="
+  + metadataType);
+  switch (metadataType) {
+  case TIMES:
+content.append(", mtime=" + mtime + ", atime=" + atime);
+break;
+  case REPLICATION:
+content.append(", replication=" + replication);
+break;
+  case OWNER:
+content.append(", ownerName=" + ownerName
++ ", groupName=" + groupName);
+break;
+  case PERMS:
+content.append(", perms=" + perms);
+break;
+  case ACLS:
+content.append(", acls=" + acls);
+break;
+  case XATTRS:
+content.append(", xAttrs=" + xAttrs + ", xAttrsRemoved="
++ xAttrsRemoved);
+break;
+  default:
+break;
+  }
+  content.append(']');
+  return content.toString();
+}
   }
 
   /**
* Sent when a file, directory, or symlink is renamed.
*/
+  @InterfaceAudience.Public
   public static class RenameEvent extends Event {
 private String srcPath;
 private String dstPath;
@@ -456,11 +520,20 @@ public abstract class Event {
 public long getTimestamp() {
   return timestamp;
 }
+
+@Override
+@InterfaceStability.Unstable
+public Stri

[32/50] [abbrv] hadoop git commit: YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. (Jonathan Yaniv and Ishai Menache via curino)

2015-07-28 Thread jianhe
http://git-wip-us.apache.org/repos/asf/hadoop/blob/156f24ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java
deleted file mode 100644
index de94dcd..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java
+++ /dev/null
@@ -1,604 +0,0 @@
-/***
- *   Licensed to the Apache Software Foundation (ASF) under one
- *   or more contributor license agreements.  See the NOTICE file
- *   distributed with this work for additional information
- *   regarding copyright ownership.  The ASF licenses this file
- *   to you under the Apache License, Version 2.0 (the
- *   "License"); you may not use this file except in compliance
- *   with the License.  You may obtain a copy of the License at
- *  
- *   http://www.apache.org/licenses/LICENSE-2.0
- *  
- *   Unless required by applicable law or agreed to in writing, software
- *   distributed under the License is distributed on an "AS IS" BASIS,
- *   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *   See the License for the specific language governing permissions and
- *   limitations under the License.
- 
***/
-package org.apache.hadoop.yarn.server.resourcemanager.reservation;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import org.apache.hadoop.yarn.api.records.ReservationDefinition;
-import org.apache.hadoop.yarn.api.records.ReservationId;
-import org.apache.hadoop.yarn.api.records.ReservationRequest;
-import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
-import org.apache.hadoop.yarn.api.records.ReservationRequests;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl;
-import 
org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
-import 
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.junit.Before;
-import org.junit.Test;
-import org.mortbay.log.Log;
-
-public class TestGreedyReservationAgent {
-
-  ReservationAgent agent;
-  InMemoryPlan plan;
-  Resource minAlloc = Resource.newInstance(1024, 1);
-  ResourceCalculator res = new DefaultResourceCalculator();
-  Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
-  Random rand = new Random();
-  long step;
-
-  @Before
-  public void setup() throws Exception {
-
-long seed = rand.nextLong();
-rand.setSeed(seed);
-Log.info("Running with seed: " + seed);
-
-// setting completely loose quotas
-long timeWindow = 100L;
-Resource clusterCapacity = Resource.newInstance(100 * 1024, 100);
-step = 1000L;
-ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
-String reservationQ = testUtil.getFullReservationQueueName();
-
-float instConstraint = 100;
-float avgConstraint = 100;
-
-ReservationSchedulerConfiguration conf =
-ReservationSystemTestUtil.createConf(reservationQ, timeWindow,
-instConstraint, avgConstraint);
-CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
-policy.init(reservationQ, conf);
-agent = new GreedyReservationAgent();
-
-QueueMetrics queueMetrics = mock(QueueMetrics.class);
-
-plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step,
-res, minAlloc, maxAlloc, "dedicated", null, true);
-  }
-
-  @SuppressWarnings("javadoc")
-  @Test
-  public void testSimple() throws PlanningException {
-
-prepareBasicPlan();
-
-// create 

[22/50] [abbrv] hadoop git commit: YARN-3969. Updating CHANGES.txt to reflect the correct set of branches where this is committed

2015-07-28 Thread jianhe
YARN-3969. Updating CHANGES.txt to reflect the correct set of branches where 
this is committed


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fc42fa8a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fc42fa8a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fc42fa8a

Branch: refs/heads/YARN-1197
Commit: fc42fa8ae3bc9d6d055090a7bb5e6f0c5972fcff
Parents: e4b0c74
Author: carlo curino 
Authored: Fri Jul 24 13:38:44 2015 -0700
Committer: carlo curino 
Committed: Fri Jul 24 13:38:44 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt | 6 --
 1 file changed, 4 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc42fa8a/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 44e5510..d1546b2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -710,6 +710,10 @@ Release 2.7.2 - UNRELEASED
 YARN-3878. AsyncDispatcher can hang while stopping if it is configured for
 draining events on stop. (Varun Saxena via jianhe)
 
+YARN-3969. Allow jobs to be submitted to reservation that is active 
+but does not have any allocations. (subru via curino)
+
+
 Release 2.7.1 - 2015-07-06 
 
   INCOMPATIBLE CHANGES
@@ -870,8 +874,6 @@ Release 2.7.1 - 2015-07-06
 YARN-3850. NM fails to read files from full disks which can lead to
 container logs being lost and other issues (Varun Saxena via jlowe)
 
-YARN-3969. Allow jobs to be submitted to reservation that is active 
-but does not have any allocations. (subru via curino)
 
 Release 2.7.0 - 2015-04-20
 



[17/50] [abbrv] hadoop git commit: HADOOP-12259. Utility to Dynamic port allocation (brahmareddy via rkanter)

2015-07-28 Thread jianhe
HADOOP-12259. Utility to Dynamic port allocation (brahmareddy via rkanter)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee233ec9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee233ec9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee233ec9

Branch: refs/heads/YARN-1197
Commit: ee233ec95ce8cfc8309d3adc072d926cd85eba08
Parents: 0fcb4a8
Author: Robert Kanter 
Authored: Fri Jul 24 09:41:53 2015 -0700
Committer: Robert Kanter 
Committed: Fri Jul 24 09:41:53 2015 -0700

--
 hadoop-common-project/hadoop-common/CHANGES.txt |  2 +
 .../org/apache/hadoop/net/ServerSocketUtil.java | 63 
 2 files changed, 65 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee233ec9/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 56edcac..d6d43f2 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -725,6 +725,8 @@ Release 2.8.0 - UNRELEASED
 HADOOP-12189. Improve CallQueueManager#swapQueue to make queue elements
 drop nearly impossible. (Zhihai Xu via wang)
 
+HADOOP-12259. Utility to Dynamic port allocation (brahmareddy via rkanter)
+
   OPTIMIZATIONS
 
 HADOOP-11785. Reduce the number of listStatus operation in distcp

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee233ec9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
new file mode 100644
index 000..0ce835f
--- /dev/null
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.net;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class ServerSocketUtil {
+
+  private static final Log LOG = LogFactory.getLog(ServerSocketUtil.class);
+
+  /**
+   * Port scan & allocate is how most other apps find ports
+   * 
+   * @param port given port
+   * @param retries number of retires
+   * @return
+   * @throws IOException
+   */
+  public static int getPort(int port, int retries) throws IOException {
+Random rand = new Random();
+int tryPort = port;
+int tries = 0;
+while (true) {
+  if (tries > 0) {
+tryPort = port + rand.nextInt(65535 - port);
+  }
+  LOG.info("Using port " + tryPort);
+  try (ServerSocket s = new ServerSocket(tryPort)) {
+return tryPort;
+  } catch (IOException e) {
+tries++;
+if (tries >= retries) {
+  LOG.info("Port is already in use; giving up");
+  throw e;
+} else {
+  LOG.info("Port is already in use; trying again");
+}
+  }
+}
+  }
+
+}



[11/50] [abbrv] hadoop git commit: YARN-3900. Protobuf layout of yarn_security_token causes errors in other protos that include it (adhoot via rkanter)

2015-07-28 Thread jianhe
YARN-3900. Protobuf layout of yarn_security_token causes errors in other protos 
that include it (adhoot via rkanter)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d3026e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d3026e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d3026e7

Branch: refs/heads/YARN-1197
Commit: 1d3026e7b3cf2f3a8a544b66ff14783cc590bdac
Parents: 6736a1a
Author: Robert Kanter 
Authored: Thu Jul 23 14:42:49 2015 -0700
Committer: Robert Kanter 
Committed: Thu Jul 23 14:46:54 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |  3 +
 .../hadoop-yarn/hadoop-yarn-common/pom.xml  |  2 +-
 .../main/proto/server/yarn_security_token.proto | 70 
 .../src/main/proto/yarn_security_token.proto| 70 
 .../pom.xml |  2 +-
 .../hadoop-yarn-server-resourcemanager/pom.xml  |  2 +-
 .../resourcemanager/recovery/TestProtos.java| 36 ++
 7 files changed, 112 insertions(+), 73 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d3026e7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9416cd6..3d41ba7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -666,6 +666,9 @@ Release 2.8.0 - UNRELEASED
 
 YARN-3941. Proportional Preemption policy should try to avoid sending 
duplicate PREEMPT_CONTAINER event to scheduler. (Sunil G via wangda)
 
+YARN-3900. Protobuf layout of yarn_security_token causes errors in other 
protos
+that include it (adhoot via rkanter)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d3026e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index 602fcd7..3b47cdd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -253,7 +253,7 @@
 ${basedir}/src/main/proto
   
   
-${basedir}/src/main/proto/server
+${basedir}/src/main/proto
 
   yarn_security_token.proto
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d3026e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto
deleted file mode 100644
index 339e99e..000
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.hadoop.yarn.proto";
-option java_outer_classname = "YarnSecurityTokenProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.yarn;
-
-import "yarn_protos.proto";
-
-// None of the following records are supposed to be exposed to users.
-
-message NMTokenIdentifierProto {
-  optional ApplicationAttemptIdProto appAttemptId = 1;
-  optional NodeIdProto nodeId = 2;
-  optional string appSubmitter = 3;
-  optional int32 keyId = 4 [default = -1];
-}
-
-message AMRMTokenIdentifierProto {
-  optional ApplicationAttemptIdProto appAttemptId = 1;
-  optional int32 keyId = 2 [default = -1];
-}
-
-message ContainerTokenIdentifierProto {
-  optional ContainerIdProto containerId = 1;
-  optional string nmHostAddr = 2;
-  option

[25/50] [abbrv] hadoop git commit: YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container log files from full disks. Contributed by zhihai xu

2015-07-28 Thread jianhe
YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container log 
files from full disks. Contributed by zhihai xu


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff9c13e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff9c13e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff9c13e0

Branch: refs/heads/YARN-1197
Commit: ff9c13e0a739bb13115167dc661b6a16b2ed2c04
Parents: 83fe34a
Author: Jason Lowe 
Authored: Fri Jul 24 22:14:39 2015 +
Committer: Jason Lowe 
Committed: Fri Jul 24 22:14:39 2015 +

--
 hadoop-yarn-project/CHANGES.txt |  2 +
 .../nodemanager/LocalDirsHandlerService.java| 35 +-
 .../webapp/TestContainerLogsPage.java   | 48 
 3 files changed, 83 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff9c13e0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cf00fe5..c295784 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -716,6 +716,8 @@ Release 2.7.2 - UNRELEASED
 YARN-3969. Allow jobs to be submitted to reservation that is active 
 but does not have any allocations. (subru via curino)
 
+YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container
+log files from full disks. (zhihai xu via jlowe)
 
 Release 2.7.1 - 2015-07-06 
 

http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff9c13e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index 0a61035..6709c90 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -18,6 +18,7 @@
 
 package org.apache.hadoop.yarn.server.nodemanager;
 
+import java.io.File;
 import java.io.IOException;
 import java.net.URI;
 import java.util.ArrayList;
@@ -31,6 +32,7 @@ import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileSystem;
 import org.apache.hadoop.fs.LocalDirAllocator;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
@@ -467,6 +469,35 @@ public class LocalDirsHandlerService extends 
AbstractService {
 return disksTurnedGood;
   }
 
+  private Path getPathToRead(String pathStr, List dirs)
+  throws IOException {
+// remove the leading slash from the path (to make sure that the uri
+// resolution results in a valid path on the dir being checked)
+if (pathStr.startsWith("/")) {
+  pathStr = pathStr.substring(1);
+}
+
+FileSystem localFS = FileSystem.getLocal(getConfig());
+for (String dir : dirs) {
+  try {
+Path tmpDir = new Path(dir);
+File tmpFile = tmpDir.isAbsolute()
+? new File(localFS.makeQualified(tmpDir).toUri())
+: new File(dir);
+Path file = new Path(tmpFile.getPath(), pathStr);
+if (localFS.exists(file)) {
+  return file;
+}
+  } catch (IOException ie) {
+// ignore
+LOG.warn("Failed to find " + pathStr + " at " + dir, ie);
+  }
+}
+
+throw new IOException("Could not find " + pathStr + " in any of" +
+" the directories");
+  }
+
   public Path getLocalPathForWrite(String pathStr) throws IOException {
 return localDirsAllocator.getLocalPathForWrite(pathStr, getConfig());
   }
@@ -484,9 +515,9 @@ public class LocalDirsHandlerService extends 
AbstractService {
   }
 
   public Path getLogPathToRead(String pathStr) throws IOException {
-return logDirsAllocator.getLocalPathToRead(pathStr, getConfig());
+return getPathToRead(pathStr, getLogDirsForRead());
   }
-  
+
   public static String[] validatePaths(String[] paths) {
 ArrayList validPaths = new ArrayList();
 for (int i = 0; i < paths.length; ++i) {

http://git-wip-us.apache.or

[38/50] [abbrv] hadoop git commit: YARN-3852. Add docker container support to container-executor. Contributed by Abin Shahab.

2015-07-28 Thread jianhe
YARN-3852. Add docker container support to container-executor. Contributed by 
Abin Shahab.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f36835ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f36835ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f36835ff

Branch: refs/heads/YARN-1197
Commit: f36835ff9b878fa20fe58a30f9d1e8c47702d6d2
Parents: 2196e39
Author: Varun Vasudev 
Authored: Mon Jul 27 10:12:30 2015 -0700
Committer: Varun Vasudev 
Committed: Mon Jul 27 10:14:51 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |   3 +
 .../container-executor/impl/configuration.c |  17 +-
 .../container-executor/impl/configuration.h |   2 +
 .../impl/container-executor.c   | 417 ---
 .../impl/container-executor.h   |  25 +-
 .../main/native/container-executor/impl/main.c  |  97 -
 6 files changed, 480 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f36835ff/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3b7d8a8..4e54aea 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -150,6 +150,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. 
 (Jonathan Yaniv and Ishai Menache via curino)
 
+YARN-3852. Add docker container support to container-executor
+(Abin Shahab via vvasudev)
+
   IMPROVEMENTS
 
 YARN-644. Basic null check is not performed on passed in arguments before

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f36835ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index eaa1f19..2825367 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -291,27 +291,23 @@ char ** get_values(const char * key) {
   return extract_values(value);
 }
 
-/**
- * Extracts array of values from the '%' separated list of values.
- */
-char ** extract_values(char *value) {
+char ** extract_values_delim(char *value, const char *delim) {
   char ** toPass = NULL;
   char *tempTok = NULL;
   char *tempstr = NULL;
   int size = 0;
   int toPassSize = MAX_SIZE;
-
   //first allocate any array of 10
   if(value != NULL) {
 toPass = (char **) malloc(sizeof(char *) * toPassSize);
-tempTok = strtok_r((char *)value, "%", &tempstr);
+tempTok = strtok_r((char *)value, delim, &tempstr);
 while (tempTok != NULL) {
   toPass[size++] = tempTok;
   if(size == toPassSize) {
 toPassSize += MAX_SIZE;
 toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize));
   }
-  tempTok = strtok_r(NULL, "%", &tempstr);
+  tempTok = strtok_r(NULL, delim, &tempstr);
 }
   }
   if (toPass != NULL) {
@@ -320,6 +316,13 @@ char ** extract_values(char *value) {
   return toPass;
 }
 
+/**
+ * Extracts array of values from the '%' separated list of values.
+ */
+char ** extract_values(char *value) {
+  extract_values_delim(value, "%");
+}
+
 // free an entry set of values
 void free_values(char** values) {
   if (*values != NULL) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f36835ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
index 133e67b..390a5b5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
@@ -46,6 +46,8 @@ char ** get_values(con

[37/50] [abbrv] hadoop git commit: HDFS-8785. TestDistributedFileSystem is failing in trunk. Contributed by Xiaoyu Yao.

2015-07-28 Thread jianhe
HDFS-8785. TestDistributedFileSystem is failing in trunk. Contributed by Xiaoyu 
Yao.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2196e39e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2196e39e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2196e39e

Branch: refs/heads/YARN-1197
Commit: 2196e39e142b0f8d1944805db2bfacd4e3244625
Parents: 1df7868
Author: Xiaoyu Yao 
Authored: Mon Jul 27 07:28:41 2015 -0700
Committer: Xiaoyu Yao 
Committed: Mon Jul 27 07:28:41 2015 -0700

--
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt|  2 ++
 .../apache/hadoop/hdfs/TestDistributedFileSystem.java  | 13 -
 2 files changed, 10 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/2196e39e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1ddf7da..cc2a833 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1084,6 +1084,8 @@ Release 2.8.0 - UNRELEASED
 HDFS-8810. Correct assertions in TestDFSInotifyEventInputStream class.
 (Surendra Singh Lilhore via aajisaka)
 
+HDFS-8785. TestDistributedFileSystem is failing in trunk. (Xiaoyu Yao)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/2196e39e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 0b77210..6012c5d 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -1189,19 +1189,22 @@ public class TestDistributedFileSystem {
 try {
   cluster.waitActive();
   DistributedFileSystem dfs = cluster.getFileSystem();
-  // Write 1 MB to a dummy socket to ensure the write times out
+  // Write 10 MB to a dummy socket to ensure the write times out
   ServerSocket socket = new ServerSocket(0);
   Peer peer = dfs.getClient().newConnectedPeer(
 (InetSocketAddress) socket.getLocalSocketAddress(), null, null);
   long start = Time.now();
   try {
-byte[] buf = new byte[1024 * 1024];
+byte[] buf = new byte[10 * 1024 * 1024];
 peer.getOutputStream().write(buf);
-Assert.fail("write should timeout");
+long delta = Time.now() - start;
+Assert.fail("write finish in " + delta + " ms" + "but should 
timedout");
   } catch (SocketTimeoutException ste) {
 long delta = Time.now() - start;
-Assert.assertTrue("write timedout too soon", delta >= timeout * 0.9);
-Assert.assertTrue("write timedout too late", delta <= timeout * 1.1);
+Assert.assertTrue("write timedout too soon in " + delta + " ms",
+delta >= timeout * 0.9);
+Assert.assertTrue("write timedout too late in " + delta + " ms",
+delta <= timeout * 1.2);
   } catch (Throwable t) {
 Assert.fail("wrong exception:" + t);
   }



[14/50] [abbrv] hadoop git commit: HDFS-6682. Add a metric to expose the timestamp of the oldest under-replicated block. (aajisaka)

2015-07-28 Thread jianhe
HDFS-6682. Add a metric to expose the timestamp of the oldest under-replicated 
block. (aajisaka)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02c01815
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02c01815
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02c01815

Branch: refs/heads/YARN-1197
Commit: 02c01815eca656814febcdaca6115e5f53b9c746
Parents: ab3197c
Author: Akira Ajisaka 
Authored: Fri Jul 24 11:37:23 2015 +0900
Committer: Akira Ajisaka 
Committed: Fri Jul 24 11:37:23 2015 +0900

--
 .../hadoop-common/src/site/markdown/Metrics.md  |  1 +
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |  3 ++
 .../server/blockmanagement/BlockManager.java|  4 ++
 .../blockmanagement/UnderReplicatedBlocks.java  | 33 --
 .../hdfs/server/namenode/FSNamesystem.java  |  9 +++-
 .../TestUnderReplicatedBlocks.java  | 48 
 6 files changed, 93 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/02c01815/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
--
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 646cda5..2e6c095 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -201,6 +201,7 @@ Each metrics record contains tags such as HAState and 
Hostname as additional inf
 | Name | Description |
 |: |: |
 | `MissingBlocks` | Current number of missing blocks |
+| `TimeOfTheOldestBlockToBeReplicated` | The timestamp of the oldest block to 
be replicated. If there are no under-replicated or corrupt blocks, return 0. |
 | `ExpiredHeartbeats` | Total number of expired heartbeats |
 | `TransactionsSinceLastCheckpoint` | Total number of transactions since last 
checkpoint |
 | `TransactionsSinceLastLogRoll` | Total number of transactions since last 
edit log roll |

http://git-wip-us.apache.org/repos/asf/hadoop/blob/02c01815/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt 
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index bcc1e25..f86d41e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -747,6 +747,9 @@ Release 2.8.0 - UNRELEASED
 HDFS-8730. Clean up the import statements in ClientProtocol.
 (Takanobu Asanuma via wheat9)
 
+HDFS-6682. Add a metric to expose the timestamp of the oldest
+under-replicated block. (aajisaka)
+
   OPTIMIZATIONS
 
 HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than

http://git-wip-us.apache.org/repos/asf/hadoop/blob/02c01815/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 7dce2a8..64603d0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -171,6 +171,10 @@ public class BlockManager implements BlockStatsMXBean {
   public int getPendingDataNodeMessageCount() {
 return pendingDNMessages.count();
   }
+  /** Used by metrics. */
+  public long getTimeOfTheOldestBlockToBeReplicated() {
+return neededReplications.getTimeOfTheOldestBlockToBeReplicated();
+  }
 
   /**replicationRecheckInterval is how often namenode checks for new 
replication work*/
   private final long replicationRecheckInterval;

http://git-wip-us.apache.org/repos/asf/hadoop/blob/02c01815/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
index 000416e..d8aec99 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/

[04/50] [abbrv] hadoop git commit: YARN-3956. Fix TestNodeManagerHardwareUtils fails on Mac (Varun Vasudev via wangda)

2015-07-28 Thread jianhe
YARN-3956. Fix TestNodeManagerHardwareUtils fails on Mac (Varun Vasudev via 
wangda)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06e5dd2c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06e5dd2c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06e5dd2c

Branch: refs/heads/YARN-1197
Commit: 06e5dd2c84c49460884757b56980b1b9c58af996
Parents: 8376ea3
Author: Wangda Tan 
Authored: Wed Jul 22 11:59:31 2015 -0700
Committer: Wangda Tan 
Committed: Wed Jul 22 12:01:41 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt | 2 ++
 .../server/nodemanager/util/TestNodeManagerHardwareUtils.java   | 5 +
 2 files changed, 7 insertions(+)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5dd2c/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index eb52745..a5fd4e7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -659,6 +659,8 @@ Release 2.8.0 - UNRELEASED
 YARN-3954. Fix 
TestYarnConfigurationFields#testCompareConfigurationClassAgainstXml.
 (varun saxena via rohithsharmaks)
 
+YARN-3956. Fix TestNodeManagerHardwareUtils fails on Mac (Varun Vasudev 
via wangda)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5dd2c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
index 5bf8cb7..84a045d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
@@ -30,6 +30,11 @@ import org.mockito.Mockito;
 public class TestNodeManagerHardwareUtils {
 
   static class TestResourceCalculatorPlugin extends ResourceCalculatorPlugin {
+
+TestResourceCalculatorPlugin() {
+  super(null);
+}
+
 @Override
 public long getVirtualMemorySize() {
   return 0;



[Hadoop Wiki] Update of "BindException" by SteveLoughran

2015-07-28 Thread Apache Wiki
Dear Wiki user,

You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change 
notification.

The "BindException" page has been changed by SteveLoughran:
https://wiki.apache.org/hadoop/BindException?action=diff&rev1=7&rev2=8

Comment:
+add the caveat that it's a local config problem

  As you cannot have more than one process listening on a TCP port, whatever is 
listening is stopping the service coming up. You will need to track down and 
stop that process, or change the service you are trying to start up to listen 
to a different port.
  
  How to track down the problem
+ 
   1. Identify which host/IP address the program is trying to use.
   1. Make sure the hostname is valid:try to ping it; use {{{ifconfig}}} to 
list the network interfaces and their IP addresses.
   1. Make sure the hostname/IP address is one belonging to the host in 
question.
@@ -24, +25 @@

   1. As root use {{{netstat -a -t --numeric-ports -p}}} to list the ports that 
are in use by number and process. (On OS/X you need to use {{{lsof}}}).
   1. Change the configuration of one of the programs to listen on a different 
port. 
  
+ Finally, this is not a Hadoop problems, it is a host, network or Hadoop 
configuration problem. As it is your cluster, [[YourNetworkYourProblem|only you 
can find out and track down the problem.]]. Sorry
+ 


[Hadoop Wiki] Update of "BindException" by SteveLoughran

2015-07-28 Thread Apache Wiki
Dear Wiki user,

You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change 
notification.

The "BindException" page has been changed by SteveLoughran:
https://wiki.apache.org/hadoop/BindException?action=diff&rev1=7&rev2=8

Comment:
move hostname up for a better workflow

  As you cannot have more than one process listening on a TCP port, whatever is 
listening is stopping the service coming up. You will need to track down and 
stop that process, or change the service you are trying to start up to listen 
to a different port.
  
  How to track down the problem
+ 
   1. Identify which host/IP address the program is trying to use.
   1. Make sure the hostname is valid:try to ping it; use {{{ifconfig}}} to 
list the network interfaces and their IP addresses.
   1. Make sure the hostname/IP address is one belonging to the host in 
question.
@@ -24, +25 @@

   1. As root use {{{netstat -a -t --numeric-ports -p}}} to list the ports that 
are in use by number and process. (On OS/X you need to use {{{lsof}}}).
   1. Change the configuration of one of the programs to listen on a different 
port. 
  
+ Finally, this is not a Hadoop problems, it is a host, network or Hadoop 
configuration problem. As it is your cluster, [[YourNetworkYourProblem|only you 
can find out and track down the problem.]]. Sorry
+ 


hadoop git commit: HADOOP-12175. FsShell must load SpanReceierHost to support tracing (Masatake Iwasaki via Colin P. McCabe)

2015-07-28 Thread cmccabe
Repository: hadoop
Updated Branches:
  refs/heads/branch-2 2ec008eb3 -> 587593566


HADOOP-12175. FsShell must load SpanReceierHost to support tracing (Masatake 
Iwasaki via Colin P. McCabe)

(cherry picked from commit 69b095730bd87706ddcbabeea6c513290ee4cdf5)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58759356
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58759356
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58759356

Branch: refs/heads/branch-2
Commit: 587593566914a0a4333c9ba87e59fc6488aec122
Parents: 2ec008e
Author: Colin Patrick Mccabe 
Authored: Tue Jul 28 12:49:38 2015 -0700
Committer: Colin Patrick Mccabe 
Committed: Tue Jul 28 12:56:36 2015 -0700

--
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../main/java/org/apache/hadoop/fs/FsShell.java | 11 -
 .../java/org/apache/hadoop/fs/TestFsShell.java  | 24 
 .../apache/hadoop/tracing/SetSpanReceiver.java  |  2 +-
 4 files changed, 38 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/58759356/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index b7e32ca..e2f79a8 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -526,6 +526,9 @@ Release 2.8.0 - UNRELEASED
 HADOOP-12245. References to misspelled REMAINING_QUATA in
 FileSystemShell.md. (Gabor Liptak via aajisaka)
 
+HADOOP-12175. FsShell must load SpanReceierHost to support tracing
+(Masatake Iwasaki via Colin P. McCabe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/58759356/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index 24b6339..e9c2f73 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.shell.Command;
 import org.apache.hadoop.fs.shell.CommandFactory;
 import org.apache.hadoop.fs.shell.FsCommand;
+import org.apache.hadoop.tracing.SpanReceiverHost;
 import org.apache.hadoop.tools.TableListing;
 import org.apache.hadoop.tracing.TraceUtils;
 import org.apache.hadoop.util.Tool;
@@ -57,6 +58,9 @@ public class FsShell extends Configured implements Tool {
   private final String usagePrefix =
 "Usage: hadoop fs [generic options]";
 
+  private SpanReceiverHost spanReceiverHost;
+  static final String SEHLL_HTRACE_PREFIX = "dfs.shell.htrace.";
+
   /**
* Default ctor with no configuration.  Be sure to invoke
* {@link #setConf(Configuration)} with a valid configuration prior
@@ -97,6 +101,8 @@ public class FsShell extends Configured implements Tool {
   commandFactory.addObject(new Usage(), "-usage");
   registerCommands(commandFactory);
 }
+this.spanReceiverHost =
+SpanReceiverHost.get(getConf(), SEHLL_HTRACE_PREFIX);
   }
 
   protected void registerCommands(CommandFactory factory) {
@@ -279,7 +285,7 @@ public class FsShell extends Configured implements Tool {
 // initialize FsShell
 init();
 traceSampler = new SamplerBuilder(TraceUtils.
-wrapHadoopConf("dfs.shell.htrace.", getConf())).build();
+wrapHadoopConf(SEHLL_HTRACE_PREFIX, getConf())).build();
 int exitCode = -1;
 if (argv.length < 1) {
   printUsage(System.err);
@@ -335,6 +341,9 @@ public class FsShell extends Configured implements Tool {
   fs.close();
   fs = null;
 }
+if (this.spanReceiverHost != null) {
+  this.spanReceiverHost.closeReceivers();
+}
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/58759356/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
index 12c8bcf..38e768f 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
@@ -18,6 +18,12 @@

hadoop git commit: HADOOP-12175. FsShell must load SpanReceierHost to support tracing (Masatake Iwasaki via Colin P. McCabe)

2015-07-28 Thread cmccabe
Repository: hadoop
Updated Branches:
  refs/heads/trunk f17093421 -> 69b095730


HADOOP-12175. FsShell must load SpanReceierHost to support tracing (Masatake 
Iwasaki via Colin P. McCabe)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69b09573
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69b09573
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69b09573

Branch: refs/heads/trunk
Commit: 69b095730bd87706ddcbabeea6c513290ee4cdf5
Parents: f170934
Author: Colin Patrick Mccabe 
Authored: Tue Jul 28 12:49:38 2015 -0700
Committer: Colin Patrick Mccabe 
Committed: Tue Jul 28 12:56:15 2015 -0700

--
 hadoop-common-project/hadoop-common/CHANGES.txt |  3 +++
 .../main/java/org/apache/hadoop/fs/FsShell.java | 11 -
 .../java/org/apache/hadoop/fs/TestFsShell.java  | 24 
 .../apache/hadoop/tracing/SetSpanReceiver.java  |  2 +-
 4 files changed, 38 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b09573/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index aeaa5b9..84e7076 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1020,6 +1020,9 @@ Release 2.8.0 - UNRELEASED
 HADOOP-12245. References to misspelled REMAINING_QUATA in
 FileSystemShell.md. (Gabor Liptak via aajisaka)
 
+HADOOP-12175. FsShell must load SpanReceierHost to support tracing
+(Masatake Iwasaki via Colin P. McCabe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b09573/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index 24b6339..e9c2f73 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configured;
 import org.apache.hadoop.fs.shell.Command;
 import org.apache.hadoop.fs.shell.CommandFactory;
 import org.apache.hadoop.fs.shell.FsCommand;
+import org.apache.hadoop.tracing.SpanReceiverHost;
 import org.apache.hadoop.tools.TableListing;
 import org.apache.hadoop.tracing.TraceUtils;
 import org.apache.hadoop.util.Tool;
@@ -57,6 +58,9 @@ public class FsShell extends Configured implements Tool {
   private final String usagePrefix =
 "Usage: hadoop fs [generic options]";
 
+  private SpanReceiverHost spanReceiverHost;
+  static final String SEHLL_HTRACE_PREFIX = "dfs.shell.htrace.";
+
   /**
* Default ctor with no configuration.  Be sure to invoke
* {@link #setConf(Configuration)} with a valid configuration prior
@@ -97,6 +101,8 @@ public class FsShell extends Configured implements Tool {
   commandFactory.addObject(new Usage(), "-usage");
   registerCommands(commandFactory);
 }
+this.spanReceiverHost =
+SpanReceiverHost.get(getConf(), SEHLL_HTRACE_PREFIX);
   }
 
   protected void registerCommands(CommandFactory factory) {
@@ -279,7 +285,7 @@ public class FsShell extends Configured implements Tool {
 // initialize FsShell
 init();
 traceSampler = new SamplerBuilder(TraceUtils.
-wrapHadoopConf("dfs.shell.htrace.", getConf())).build();
+wrapHadoopConf(SEHLL_HTRACE_PREFIX, getConf())).build();
 int exitCode = -1;
 if (argv.length < 1) {
   printUsage(System.err);
@@ -335,6 +341,9 @@ public class FsShell extends Configured implements Tool {
   fs.close();
   fs = null;
 }
+if (this.spanReceiverHost != null) {
+  this.spanReceiverHost.closeReceivers();
+}
   }
 
   /**

http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b09573/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
index 12c8bcf..38e768f 100644
--- 
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
+++ 
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
@@ -18,6 +18,12 @@
 package org.apache.hadoop.fs;
 
 import junit.framework.AssertionFailedE

[3/6] hadoop git commit: HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)

2015-07-28 Thread aengineer
HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/030fcfa9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/030fcfa9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/030fcfa9

Branch: refs/heads/HDFS-7240
Commit: 030fcfa99c345ad57625486eeabedebf2fd4411f
Parents: e21dde5
Author: Arun Suresh 
Authored: Mon Jul 27 23:02:03 2015 -0700
Committer: Arun Suresh 
Committed: Mon Jul 27 23:02:03 2015 -0700

--
 .../apache/hadoop/io/retry/MultiException.java  |  49 +++
 .../hadoop/io/retry/RetryInvocationHandler.java |  99 +-
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |   2 +
 .../ha/ConfiguredFailoverProxyProvider.java |  52 ++-
 .../ha/RequestHedgingProxyProvider.java | 186 ++
 .../markdown/HDFSHighAvailabilityWithNFS.md |   9 +-
 .../markdown/HDFSHighAvailabilityWithQJM.md |  10 +-
 .../ha/TestRequestHedgingProxyProvider.java | 350 +++
 8 files changed, 724 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/030fcfa9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
new file mode 100644
index 000..4963a2d
--- /dev/null
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.io.retry;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Holder class that clients can use to return multiple exceptions.
+ */
+public class MultiException extends IOException {
+
+  private final Map exes;
+
+  public MultiException(Map exes) {
+this.exes = exes;
+  }
+
+  public Map getExceptions() {
+return exes;
+  }
+
+  @Override
+  public String toString() {
+StringBuilder sb = new StringBuilder("{");
+for (Exception e : exes.values()) {
+  sb.append(e.toString()).append(", ");
+}
+sb.append("}");
+return "MultiException[" + sb.toString() + "]";
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/030fcfa9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
index 543567e..9256356 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
@@ -23,6 +23,8 @@ import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.logging.Log;
@@ -101,7 +103,7 @@ public class RetryInvocationHandler implements 
RpcInvocationHandler {
 Object ret = invokeMethod(method, args);
 hasMadeASuccessfulCall = true;
 return ret;
-  } catch (Exception e) {
+  } catch (Exception ex) {
 boolean isIdempotentOrAtMostOnce = proxyProvider.getInterface()
 .getMethod(method.getName(), method.getParameterTypes())
 .isAnnotationPresent(Idempotent.class);
@@ -110,15 +112,16 @@ public class RetryInvocationHandler implements 
RpcInvocationHandler {
   .getMethod(method.getName(), method.getParameterTypes())
   .isA

[5/6] hadoop git commit: Merge branch 'trunk' into HDFS-7240

2015-07-28 Thread aengineer
Merge branch 'trunk' into HDFS-7240


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/188d283a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/188d283a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/188d283a

Branch: refs/heads/HDFS-7240
Commit: 188d283a4debe2151ee3327ac23bad86b9b6aba3
Parents: 2ebe8c7 f170934
Author: Anu Engineer 
Authored: Tue Jul 28 11:12:08 2015 -0700
Committer: Anu Engineer 
Committed: Tue Jul 28 11:12:08 2015 -0700

--
 hadoop-common-project/hadoop-common/CHANGES.txt |   3 +
 .../apache/hadoop/io/retry/MultiException.java  |  49 +++
 .../hadoop/io/retry/RetryInvocationHandler.java |  99 +-
 .../src/site/markdown/FileSystemShell.md|   2 +-
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |   2 +
 .../ha/ConfiguredFailoverProxyProvider.java |  52 ++-
 .../ha/RequestHedgingProxyProvider.java | 186 ++
 .../markdown/HDFSHighAvailabilityWithNFS.md |   9 +-
 .../markdown/HDFSHighAvailabilityWithQJM.md |  10 +-
 .../ha/TestRequestHedgingProxyProvider.java | 350 +++
 hadoop-yarn-project/CHANGES.txt |   6 +
 .../container-executor/impl/configuration.c |   4 +-
 .../test/test-container-executor.c  |  22 +-
 .../webapp/CapacitySchedulerPage.java   |   5 +-
 14 files changed, 755 insertions(+), 44 deletions(-)
--




[6/6] hadoop git commit: HDFS-8695. OzoneHandler : Add Bucket REST Interface. (aengineer)

2015-07-28 Thread aengineer
HDFS-8695. OzoneHandler : Add Bucket REST Interface. (aengineer)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7851874
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7851874
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7851874

Branch: refs/heads/HDFS-7240
Commit: c78518749e431684cfc7685e1cf3299751b130e6
Parents: 188d283
Author: Anu Engineer 
Authored: Tue Jul 28 11:13:13 2015 -0700
Committer: Anu Engineer 
Committed: Tue Jul 28 11:13:13 2015 -0700

--
 .../hadoop/ozone/web/exceptions/ErrorTable.java |   5 +
 .../hadoop/ozone/web/handlers/BucketArgs.java   |  10 -
 .../ozone/web/handlers/BucketHandler.java   | 193 +
 .../web/handlers/BucketProcessTemplate.java | 278 +++
 .../apache/hadoop/ozone/web/headers/Header.java |  14 +-
 .../hadoop/ozone/web/interfaces/Bucket.java | 133 +
 .../ozone/web/interfaces/StorageHandler.java|  85 ++
 .../web/localstorage/LocalStorageHandler.java   | 109 
 8 files changed, 815 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7851874/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
index a51dac5..7e75cf0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
@@ -58,6 +58,7 @@ public final class ErrorTable {
   new OzoneException(HTTP_BAD_REQUEST, "malformedACL",
  "Invalid ACL specified.");
 
+
   public static final OzoneException INVALID_VOLUME_NAME =
   new OzoneException(HTTP_BAD_REQUEST, "invalidVolumeName",
  "Invalid volume name.");
@@ -81,6 +82,10 @@ public final class ErrorTable {
   new OzoneException(HTTP_BAD_REQUEST, "malformedBucketVersion",
  "Malformed bucket version or version not unique.");
 
+  public static final OzoneException MALFORMED_STORAGE_TYPE =
+  new OzoneException(HTTP_BAD_REQUEST, "malformedStorageType",
+ "Invalid storage Type specified.");
+
   public static final OzoneException MALFORMED_STORAGE_CLASS =
   new OzoneException(HTTP_BAD_REQUEST, "malformedStorageClass",
  "Invalid storage class specified.");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7851874/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
index 315ae3f..d62c72d 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
@@ -164,16 +164,6 @@ public class BucketArgs extends VolumeArgs {
 return versioning;
   }
 
-  /**
-   * Converts a valid String to Enum for ease of use.
-   *
-   * @param version version string.
-   */
-  public void setVersioning(String version) {
-if (version != null) {
-  this.versioning = OzoneConsts.Versioning.valueOf(version.toUpperCase());
-}
-  }
 
   /**
* SetVersioning Info.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7851874/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
new file mode 100644
index 000..2005367
--- /dev/null
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
@@ -0,0 +1,193 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except

[1/6] hadoop git commit: YARN-3846. RM Web UI queue filter is not working for sub queue. Contributed by Mohammad Shahid Khan

2015-07-28 Thread aengineer
Repository: hadoop
Updated Branches:
  refs/heads/HDFS-7240 2ebe8c7cb -> c78518749


YARN-3846. RM Web UI queue filter is not working for sub queue. Contributed by 
Mohammad Shahid Khan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3572ebd7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3572ebd7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3572ebd7

Branch: refs/heads/HDFS-7240
Commit: 3572ebd738aa5fa8b0906d75fb12cc6cbb991573
Parents: 3e6fce9
Author: Jian He 
Authored: Mon Jul 27 16:57:11 2015 -0700
Committer: Jian He 
Committed: Mon Jul 27 17:12:05 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt | 3 +++
 .../server/resourcemanager/webapp/CapacitySchedulerPage.java| 5 -
 2 files changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3572ebd7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 534c55a..4f8484a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -695,6 +695,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3958. TestYarnConfigurationFields should be moved to hadoop-yarn-api
 module. (Varun Saxena via aajisaka)
 
+YARN-3846. RM Web UI queue filter is not working for sub queue.
+(Mohammad Shahid Khan via jianhe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3572ebd7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 12a3013..d8971b7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -516,7 +516,10 @@ class CapacitySchedulerPage extends RmView {
   "  $('#cs').bind('select_node.jstree', function(e, data) {",
   "var q = $('.q', data.rslt.obj).first().text();",
   "if (q == 'Queue: root') q = '';",
-  "else q = '^' + q.substr(q.lastIndexOf(':') + 2) + '$';",
+  "else {",
+  "  q = q.substr(q.lastIndexOf(':') + 2);",
+  "  q = '^' + q.substr(q.lastIndexOf('.') + 1) + '$';",
+  "}",
   "$('#apps').dataTable().fnFilter(q, 4, true);",
   "  });",
   "  $('#cs').show();",



[2/6] hadoop git commit: HADOOP-12245. References to misspelled REMAINING_QUATA in FileSystemShell.md. Contributed by Gabor Liptak.

2015-07-28 Thread aengineer
HADOOP-12245. References to misspelled REMAINING_QUATA in FileSystemShell.md. 
Contributed by Gabor Liptak.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e21dde50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e21dde50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e21dde50

Branch: refs/heads/HDFS-7240
Commit: e21dde501aa9323b7f34b4bc4ba9d282ec4f2707
Parents: 3572ebd
Author: Akira Ajisaka 
Authored: Tue Jul 28 11:33:10 2015 +0900
Committer: Akira Ajisaka 
Committed: Tue Jul 28 11:33:10 2015 +0900

--
 hadoop-common-project/hadoop-common/CHANGES.txt   | 3 +++
 .../hadoop-common/src/site/markdown/FileSystemShell.md| 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e21dde50/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index baf39e3..aeaa5b9 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1017,6 +1017,9 @@ Release 2.8.0 - UNRELEASED
 HADOOP-12239. StorageException complaining " no lease ID" when updating
 FolderLastModifiedTime in WASB. (Duo Xu via cnauroth)
 
+HADOOP-12245. References to misspelled REMAINING_QUATA in
+FileSystemShell.md. (Gabor Liptak via aajisaka)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e21dde50/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
--
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 144cb73..fb89ca1 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -174,7 +174,7 @@ Usage: `hadoop fs -count [-q] [-h] [-v]  `
 
 Count the number of directories, files and bytes under the paths that match 
the specified file pattern. The output columns with -count are: DIR\_COUNT, 
FILE\_COUNT, CONTENT\_SIZE, PATHNAME
 
-The output columns with -count -q are: QUOTA, REMAINING\_QUATA, SPACE\_QUOTA, 
REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
+The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, 
REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
 
 The -h option shows sizes in human readable format.
 



[4/6] hadoop git commit: YARN-3982. container-executor parsing of container-executor.cfg broken in trunk and branch-2. Contributed by Varun Vasudev

2015-07-28 Thread aengineer
YARN-3982. container-executor parsing of container-executor.cfg broken
in trunk and branch-2. Contributed by Varun Vasudev


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f1709342
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f1709342
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f1709342

Branch: refs/heads/HDFS-7240
Commit: f17093421521efcbdc813f6f2b8411e45ecc7863
Parents: 030fcfa
Author: Xuan 
Authored: Mon Jul 27 23:45:58 2015 -0700
Committer: Xuan 
Committed: Mon Jul 27 23:45:58 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |  3 +++
 .../container-executor/impl/configuration.c |  4 ++--
 .../test/test-container-executor.c  | 22 +---
 3 files changed, 20 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4f8484a..b4666e8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -698,6 +698,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3846. RM Web UI queue filter is not working for sub queue.
 (Mohammad Shahid Khan via jianhe)
 
+YARN-3982. container-executor parsing of container-executor.cfg broken in
+trunk and branch-2. (Varun Vasudev via xgong)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index 2825367..373dbfd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -284,11 +284,11 @@ char * get_value(const char* key) {
 
 /**
  * Function to return an array of values for a key.
- * Value delimiter is assumed to be a '%'.
+ * Value delimiter is assumed to be a ','.
  */
 char ** get_values(const char * key) {
   char *value = get_value(key);
-  return extract_values(value);
+  return extract_values_delim(value, ",");
 }
 
 char ** extract_values_delim(char *value, const char *delim) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 99bcf34..001a37d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -145,7 +145,7 @@ void check_pid_file(const char* pid_file, pid_t mypid) {
   }
 
   char myPidBuf[33];
-  snprintf(myPidBuf, 33, "%" PRId64, (int64_t)mypid);
+  snprintf(myPidBuf, 33, "%" PRId64, (int64_t)(mypid + 1));
   if (strncmp(pidBuf, myPidBuf, strlen(myPidBuf)) != 0) {
 printf("FAIL: failed to find matching pid in pid file\n");
 printf("FAIL: Expected pid %" PRId64 " : Got %.*s", (int64_t)mypid,
@@ -212,15 +212,15 @@ void test_get_app_log_dir() {
   free(logdir);
 }
 
-void test_check_user() {
+void test_check_user(int expectedFailure) {
   printf("\nTesting test_check_user\n");
   struct passwd *user = check_user(username);
-  if (user == NULL) {
+  if (user == NULL && !expectedFailure) {
 printf("FAIL: failed check for user %s\n", username);
 exit(1);
   }
   free(user);
-  if (check_user("lp") != NULL) {
+  if (check_user("lp") != NULL && !expectedFailure) {
 printf("FAIL: failed check for system user lp\n");
 exit(1);
   }
@@ -228,7 +228,7 @@ void test_check_user() {
 printf("FAIL: failed check for system user root\n

Git Push Summary

2015-07-28 Thread aengineer
Repository: hadoop
Updated Branches:
  refs/heads/hdfs-7240 [deleted] be57a1345


[2/2] hadoop git commit: HDFS-8695. OzoneHandler : Add Bucket REST Interface. (aengineer)

2015-07-28 Thread aengineer
HDFS-8695. OzoneHandler : Add Bucket REST Interface. (aengineer)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be57a134
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be57a134
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be57a134

Branch: refs/heads/hdfs-7240
Commit: be57a1345fce613093692b3da0e8615412cc0fdc
Parents: 6b3866d
Author: Anu Engineer 
Authored: Tue Jul 28 10:56:11 2015 -0700
Committer: Anu Engineer 
Committed: Tue Jul 28 10:56:11 2015 -0700

--
 .../hadoop/ozone/web/exceptions/ErrorTable.java |   5 +
 .../hadoop/ozone/web/handlers/BucketArgs.java   |  10 -
 .../ozone/web/handlers/BucketHandler.java   | 193 +
 .../web/handlers/BucketProcessTemplate.java | 278 +++
 .../apache/hadoop/ozone/web/headers/Header.java |  14 +-
 .../hadoop/ozone/web/interfaces/Bucket.java | 133 +
 .../ozone/web/interfaces/StorageHandler.java|  85 ++
 .../web/localstorage/LocalStorageHandler.java   | 109 
 8 files changed, 815 insertions(+), 12 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/be57a134/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
index a51dac5..7e75cf0 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
@@ -58,6 +58,7 @@ public final class ErrorTable {
   new OzoneException(HTTP_BAD_REQUEST, "malformedACL",
  "Invalid ACL specified.");
 
+
   public static final OzoneException INVALID_VOLUME_NAME =
   new OzoneException(HTTP_BAD_REQUEST, "invalidVolumeName",
  "Invalid volume name.");
@@ -81,6 +82,10 @@ public final class ErrorTable {
   new OzoneException(HTTP_BAD_REQUEST, "malformedBucketVersion",
  "Malformed bucket version or version not unique.");
 
+  public static final OzoneException MALFORMED_STORAGE_TYPE =
+  new OzoneException(HTTP_BAD_REQUEST, "malformedStorageType",
+ "Invalid storage Type specified.");
+
   public static final OzoneException MALFORMED_STORAGE_CLASS =
   new OzoneException(HTTP_BAD_REQUEST, "malformedStorageClass",
  "Invalid storage class specified.");

http://git-wip-us.apache.org/repos/asf/hadoop/blob/be57a134/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
index 315ae3f..d62c72d 100644
--- 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
@@ -164,16 +164,6 @@ public class BucketArgs extends VolumeArgs {
 return versioning;
   }
 
-  /**
-   * Converts a valid String to Enum for ease of use.
-   *
-   * @param version version string.
-   */
-  public void setVersioning(String version) {
-if (version != null) {
-  this.versioning = OzoneConsts.Versioning.valueOf(version.toUpperCase());
-}
-  }
 
   /**
* SetVersioning Info.

http://git-wip-us.apache.org/repos/asf/hadoop/blob/be57a134/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
--
diff --git 
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
new file mode 100644
index 000..2005367
--- /dev/null
+++ 
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
@@ -0,0 +1,193 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except

[1/2] hadoop git commit: Merge branch 'trunk' into hdfs-7240

2015-07-28 Thread aengineer
Repository: hadoop
Updated Branches:
  refs/heads/hdfs-7240 [created] be57a1345


Merge branch 'trunk' into hdfs-7240


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b3866d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b3866d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b3866d5

Branch: refs/heads/hdfs-7240
Commit: 6b3866d5299ad305fad4cbb576bb54dfea71e813
Parents: 2ebe8c7 f170934
Author: Anu Engineer 
Authored: Tue Jul 28 10:55:34 2015 -0700
Committer: Anu Engineer 
Committed: Tue Jul 28 10:55:34 2015 -0700

--
 hadoop-common-project/hadoop-common/CHANGES.txt |   3 +
 .../apache/hadoop/io/retry/MultiException.java  |  49 +++
 .../hadoop/io/retry/RetryInvocationHandler.java |  99 +-
 .../src/site/markdown/FileSystemShell.md|   2 +-
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |   2 +
 .../ha/ConfiguredFailoverProxyProvider.java |  52 ++-
 .../ha/RequestHedgingProxyProvider.java | 186 ++
 .../markdown/HDFSHighAvailabilityWithNFS.md |   9 +-
 .../markdown/HDFSHighAvailabilityWithQJM.md |  10 +-
 .../ha/TestRequestHedgingProxyProvider.java | 350 +++
 hadoop-yarn-project/CHANGES.txt |   6 +
 .../container-executor/impl/configuration.c |   4 +-
 .../test/test-container-executor.c  |  22 +-
 .../webapp/CapacitySchedulerPage.java   |   5 +-
 14 files changed, 755 insertions(+), 44 deletions(-)
--




[02/10] hadoop git commit: YARN-3853. Add docker container runtime support to LinuxContainterExecutor. Contributed by Sidharta Seethana.

2015-07-28 Thread aw
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
new file mode 100644
index 000..f9a890e
--- /dev/null
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -0,0 +1,107 @@
+/*
+ * *
+ *  Licensed to the Apache Software Foundation (ASF) under one
+ *  or more contributor license agreements.  See the NOTICE file
+ *  distributed with this work for additional information
+ *  regarding copyright ownership.  The ASF licenses this file
+ *  to you under the Apache License, Version 2.0 (the
+ *  "License"); you may not use this file except in compliance
+ *  with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ * /
+ */
+
+package 
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+
+import org.apache.hadoop.util.StringUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DockerRunCommand extends DockerCommand {
+  private static final String RUN_COMMAND = "run";
+  private final String image;
+  private List overrrideCommandWithArgs;
+
+  /** The following are mandatory: */
+  public DockerRunCommand(String containerId, String user, String image) {
+super(RUN_COMMAND);
+super.addCommandArguments("--name=" + containerId, "--user=" + user);
+this.image = image;
+  }
+
+  public DockerRunCommand removeContainerOnExit() {
+super.addCommandArguments("--rm");
+return this;
+  }
+
+  public DockerRunCommand detachOnRun() {
+super.addCommandArguments("-d");
+return this;
+  }
+
+  public DockerRunCommand setContainerWorkDir(String workdir) {
+super.addCommandArguments("--workdir=" + workdir);
+return this;
+  }
+
+  public DockerRunCommand setNetworkType(String type) {
+super.addCommandArguments("--net=" + type);
+return this;
+  }
+
+  public DockerRunCommand addMountLocation(String sourcePath, String
+  destinationPath) {
+super.addCommandArguments("-v", sourcePath + ":" + destinationPath);
+return this;
+  }
+
+  public DockerRunCommand setCGroupParent(String parentPath) {
+super.addCommandArguments("--cgroup-parent=" + parentPath);
+return this;
+  }
+
+  public DockerRunCommand addDevice(String sourceDevice, String
+  destinationDevice) {
+super.addCommandArguments("--device=" + sourceDevice + ":" +
+destinationDevice);
+return this;
+  }
+
+  public DockerRunCommand enableDetach() {
+super.addCommandArguments("--detach=true");
+return this;
+  }
+
+  public DockerRunCommand disableDetach() {
+super.addCommandArguments("--detach=false");
+return this;
+  }
+
+  public DockerRunCommand setOverrideCommandWithArgs(
+  List overrideCommandWithArgs) {
+this.overrrideCommandWithArgs = overrideCommandWithArgs;
+return this;
+  }
+
+  @Override
+  public String getCommandWithArguments() {
+List argList = new ArrayList<>();
+
+argList.add(super.getCommandWithArguments());
+argList.add(image);
+
+if (overrrideCommandWithArgs != null) {
+  argList.addAll(overrrideCommandWithArgs);
+}
+
+return StringUtils.join(" ", argList);
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerExecutionException.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerExecutionException.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containe

[03/10] hadoop git commit: YARN-3853. Add docker container runtime support to LinuxContainterExecutor. Contributed by Sidharta Seethana.

2015-07-28 Thread aw
YARN-3853. Add docker container runtime support to LinuxContainterExecutor. 
Contributed by Sidharta Seethana.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e6fce91
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e6fce91
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e6fce91

Branch: refs/heads/HADOOP-12111
Commit: 3e6fce91a471b4a5099de109582e7c6417e8a822
Parents: f36835f
Author: Varun Vasudev 
Authored: Mon Jul 27 11:57:40 2015 -0700
Committer: Varun Vasudev 
Committed: Mon Jul 27 11:57:40 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |   4 +
 .../server/nodemanager/ContainerExecutor.java   |  23 +-
 .../nodemanager/DefaultContainerExecutor.java   |   2 +-
 .../nodemanager/DockerContainerExecutor.java|   2 +-
 .../nodemanager/LinuxContainerExecutor.java | 222 +++
 .../launcher/ContainerLaunch.java   |  15 +
 .../linux/privileged/PrivilegedOperation.java   |  46 +++-
 .../PrivilegedOperationException.java   |  30 +-
 .../privileged/PrivilegedOperationExecutor.java |  30 +-
 .../linux/resources/CGroupsHandler.java |   8 +
 .../linux/resources/CGroupsHandlerImpl.java |  12 +-
 .../runtime/DefaultLinuxContainerRuntime.java   | 148 ++
 .../DelegatingLinuxContainerRuntime.java| 110 
 .../runtime/DockerLinuxContainerRuntime.java| 273 +++
 .../linux/runtime/LinuxContainerRuntime.java|  38 +++
 .../runtime/LinuxContainerRuntimeConstants.java |  69 +
 .../linux/runtime/docker/DockerClient.java  |  82 ++
 .../linux/runtime/docker/DockerCommand.java |  66 +
 .../linux/runtime/docker/DockerLoadCommand.java |  30 ++
 .../linux/runtime/docker/DockerRunCommand.java  | 107 
 .../runtime/ContainerExecutionException.java|  85 ++
 .../runtime/ContainerRuntime.java   |  50 
 .../runtime/ContainerRuntimeConstants.java  |  33 +++
 .../runtime/ContainerRuntimeContext.java| 105 +++
 .../executor/ContainerLivenessContext.java  |  13 +
 .../executor/ContainerReacquisitionContext.java |  13 +
 .../executor/ContainerSignalContext.java|  13 +
 .../executor/ContainerStartContext.java |  23 +-
 .../TestLinuxContainerExecutorWithMocks.java| 118 +---
 .../TestPrivilegedOperationExecutor.java|   8 +-
 .../runtime/TestDockerContainerRuntime.java | 219 +++
 31 files changed, 1815 insertions(+), 182 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4e54aea..534c55a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -153,6 +153,10 @@ Release 2.8.0 - UNRELEASED
 YARN-3852. Add docker container support to container-executor
 (Abin Shahab via vvasudev)
 
+YARN-3853. Add docker container runtime support to LinuxContainterExecutor.
+(Sidharta Seethana via vvasudev)
+
+
   IMPROVEMENTS
 
 YARN-644. Basic null check is not performed on passed in arguments before

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 79f9b0d..68bfbbf 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -24,8 +24,10 @@ import java.io.OutputStream;
 import java.io.PrintStream;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -39,6 +41,7 @@ import org.apache.hadoop.conf.Configurable;
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.fs.Path;
 import org.apache.hadoop.fs.permission.FsPermission;
+import org.a

[04/10] hadoop git commit: YARN-3846. RM Web UI queue filter is not working for sub queue. Contributed by Mohammad Shahid Khan

2015-07-28 Thread aw
YARN-3846. RM Web UI queue filter is not working for sub queue. Contributed by 
Mohammad Shahid Khan


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3572ebd7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3572ebd7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3572ebd7

Branch: refs/heads/HADOOP-12111
Commit: 3572ebd738aa5fa8b0906d75fb12cc6cbb991573
Parents: 3e6fce9
Author: Jian He 
Authored: Mon Jul 27 16:57:11 2015 -0700
Committer: Jian He 
Committed: Mon Jul 27 17:12:05 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt | 3 +++
 .../server/resourcemanager/webapp/CapacitySchedulerPage.java| 5 -
 2 files changed, 7 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/3572ebd7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 534c55a..4f8484a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -695,6 +695,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3958. TestYarnConfigurationFields should be moved to hadoop-yarn-api
 module. (Varun Saxena via aajisaka)
 
+YARN-3846. RM Web UI queue filter is not working for sub queue.
+(Mohammad Shahid Khan via jianhe)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/3572ebd7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 12a3013..d8971b7 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -516,7 +516,10 @@ class CapacitySchedulerPage extends RmView {
   "  $('#cs').bind('select_node.jstree', function(e, data) {",
   "var q = $('.q', data.rslt.obj).first().text();",
   "if (q == 'Queue: root') q = '';",
-  "else q = '^' + q.substr(q.lastIndexOf(':') + 2) + '$';",
+  "else {",
+  "  q = q.substr(q.lastIndexOf(':') + 2);",
+  "  q = '^' + q.substr(q.lastIndexOf('.') + 1) + '$';",
+  "}",
   "$('#apps').dataTable().fnFilter(q, 4, true);",
   "  });",
   "  $('#cs').show();",



[08/10] hadoop git commit: Merge branch 'trunk' into HADOOP-12111

2015-07-28 Thread aw
Merge branch 'trunk' into HADOOP-12111


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b6953a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b6953a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b6953a9

Branch: refs/heads/HADOOP-12111
Commit: 0b6953a9405a1fff7360da8a22174987c2ad9936
Parents: 4d4f288 f170934
Author: Allen Wittenauer 
Authored: Tue Jul 28 10:42:31 2015 -0700
Committer: Allen Wittenauer 
Committed: Tue Jul 28 10:42:31 2015 -0700

--
 hadoop-common-project/hadoop-common/CHANGES.txt |   3 +
 .../apache/hadoop/io/retry/MultiException.java  |  49 +++
 .../hadoop/io/retry/RetryInvocationHandler.java |  99 -
 .../src/site/markdown/FileSystemShell.md|   2 +-
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |   2 +
 .../ha/ConfiguredFailoverProxyProvider.java |  52 ++-
 .../ha/RequestHedgingProxyProvider.java | 186 +
 .../markdown/HDFSHighAvailabilityWithNFS.md |   9 +-
 .../markdown/HDFSHighAvailabilityWithQJM.md |  10 +-
 .../ha/TestRequestHedgingProxyProvider.java | 350 
 hadoop-yarn-project/CHANGES.txt |  13 +
 .../server/nodemanager/ContainerExecutor.java   |  23 +-
 .../nodemanager/DefaultContainerExecutor.java   |   2 +-
 .../nodemanager/DockerContainerExecutor.java|   2 +-
 .../nodemanager/LinuxContainerExecutor.java | 222 +-
 .../launcher/ContainerLaunch.java   |  15 +
 .../linux/privileged/PrivilegedOperation.java   |  46 +-
 .../PrivilegedOperationException.java   |  30 +-
 .../privileged/PrivilegedOperationExecutor.java |  30 +-
 .../linux/resources/CGroupsHandler.java |   8 +
 .../linux/resources/CGroupsHandlerImpl.java |  12 +-
 .../runtime/DefaultLinuxContainerRuntime.java   | 148 +++
 .../DelegatingLinuxContainerRuntime.java| 110 +
 .../runtime/DockerLinuxContainerRuntime.java| 273 
 .../linux/runtime/LinuxContainerRuntime.java|  38 ++
 .../runtime/LinuxContainerRuntimeConstants.java |  69 +++
 .../linux/runtime/docker/DockerClient.java  |  82 
 .../linux/runtime/docker/DockerCommand.java |  66 +++
 .../linux/runtime/docker/DockerLoadCommand.java |  30 ++
 .../linux/runtime/docker/DockerRunCommand.java  | 107 +
 .../runtime/ContainerExecutionException.java|  85 
 .../runtime/ContainerRuntime.java   |  50 +++
 .../runtime/ContainerRuntimeConstants.java  |  33 ++
 .../runtime/ContainerRuntimeContext.java| 105 +
 .../executor/ContainerLivenessContext.java  |  13 +
 .../executor/ContainerReacquisitionContext.java |  13 +
 .../executor/ContainerSignalContext.java|  13 +
 .../executor/ContainerStartContext.java |  23 +-
 .../container-executor/impl/configuration.c |  21 +-
 .../container-executor/impl/configuration.h |   2 +
 .../impl/container-executor.c   | 417 ---
 .../impl/container-executor.h   |  25 +-
 .../main/native/container-executor/impl/main.c  |  97 -
 .../test/test-container-executor.c  |  22 +-
 .../TestLinuxContainerExecutorWithMocks.java| 118 --
 .../TestPrivilegedOperationExecutor.java|   8 +-
 .../runtime/TestDockerContainerRuntime.java | 219 ++
 .../webapp/CapacitySchedulerPage.java   |   5 +-
 48 files changed, 3050 insertions(+), 307 deletions(-)
--




[10/10] hadoop git commit: HADOOP-12254. test-patch.sh should run findbugs if only findbugs-exclude.xml has changed (Kengo Seki via aw)

2015-07-28 Thread aw
HADOOP-12254. test-patch.sh should run findbugs if only findbugs-exclude.xml 
has changed (Kengo Seki via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03335bb4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03335bb4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03335bb4

Branch: refs/heads/HADOOP-12111
Commit: 03335bb4d5a047569519ee6775e6edd0c939cf03
Parents: 7c92f0f
Author: Allen Wittenauer 
Authored: Tue Jul 28 10:44:37 2015 -0700
Committer: Allen Wittenauer 
Committed: Tue Jul 28 10:44:37 2015 -0700

--
 dev-support/test-patch.d/findbugs.sh | 19 ++-
 1 file changed, 10 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/03335bb4/dev-support/test-patch.d/findbugs.sh
--
diff --git a/dev-support/test-patch.d/findbugs.sh 
b/dev-support/test-patch.d/findbugs.sh
index 1d7118b..4fa5428 100755
--- a/dev-support/test-patch.d/findbugs.sh
+++ b/dev-support/test-patch.d/findbugs.sh
@@ -20,13 +20,14 @@ FINDBUGS_WARNINGS_FAIL_PRECHECK=false
 
 add_plugin findbugs
 
-function findbugs_file_filter
+function findbugs_filefilter
 {
   local filename=$1
 
   if [[ ${BUILDTOOL} == maven
 || ${BUILDTOOL} == ant ]]; then
-if [[ ${filename} =~ \.java$ ]]; then
+if [[ ${filename} =~ \.java$
+  || ${filename} =~ (^|/)findbugs-exclude.xml$ ]]; then
   add_test findbugs
 fi
   fi
@@ -139,7 +140,7 @@ function findbugs_runner
   savestop=$(stop_clock)
   MODULE_STATUS_TIMER[${i}]=${savestop}
   module_status ${i} -1 "" "${name}/${module} cannot run 
setBugDatabaseInfo from findbugs"
-  ((retval = retval + 1))
+  ((result=result+1))
   ((i=i+1))
   continue
 fi
@@ -151,7 +152,7 @@ function findbugs_runner
   savestop=$(stop_clock)
   MODULE_STATUS_TIMER[${i}]=${savestop}
   module_status ${i} -1 "" "${name}/${module} cannot run convertXmlToText 
from findbugs"
-  ((result = result + 1))
+  ((result=result+1))
 fi
 
 if [[ -z ${FINDBUGS_VERSION}
@@ -182,7 +183,7 @@ function findbugs_preapply
   local i=0
   local warnings_file
   local module_findbugs_warnings
-  local results=0
+  local result=0
 
   big_console_header "Pre-patch findbugs detection"
 
@@ -199,7 +200,7 @@ function findbugs_preapply
   fi
 
   findbugs_runner branch
-  results=$?
+  result=$?
 
   if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then
 until [[ $i -eq ${#MODULE[@]} ]]; do
@@ -222,7 +223,7 @@ function findbugs_preapply
 
   if [[ ${module_findbugs_warnings} -gt 0 ]] ; then
 module_status ${i} -1 "branch-findbugs-${fn}.html" "${module} in 
${PATCH_BRANCH} cannot run convertXmlToText from findbugs"
-((results=results+1))
+((result=result+1))
   fi
   savestop=$(stop_clock)
   MODULE_STATUS_TIMER[${i}]=${savestop}
@@ -231,7 +232,7 @@ function findbugs_preapply
 modules_messages branch findbugs true
   fi
 
-  if [[ ${results} != 0 ]]; then
+  if [[ ${result} != 0 ]]; then
 return 1
   fi
   return 0
@@ -256,7 +257,7 @@ function findbugs_postinstall
   local firstpart
   local secondpart
   local i=0
-  local results=0
+  local result=0
   local savestop
 
   big_console_header "Patch findbugs detection"



[09/10] hadoop git commit: HADOOP-12273. releasedocmaker.py fails with stacktrace if --project option is not specified (Kengo Seki via aw)

2015-07-28 Thread aw
HADOOP-12273. releasedocmaker.py fails with stacktrace if --project option is 
not specified (Kengo Seki via aw)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c92f0fe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c92f0fe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c92f0fe

Branch: refs/heads/HADOOP-12111
Commit: 7c92f0fe9176c55f7c16bc20211c854cdbfe0141
Parents: 0b6953a
Author: Allen Wittenauer 
Authored: Tue Jul 28 10:43:22 2015 -0700
Committer: Allen Wittenauer 
Committed: Tue Jul 28 10:43:22 2015 -0700

--
 dev-support/releasedocmaker.py | 8 ++--
 1 file changed, 2 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c92f0fe/dev-support/releasedocmaker.py
--
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index d2e5dda..c59ae99 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -400,12 +400,6 @@ def main():
   (options, args) = parser.parse_args()
 
   if (options.versions is None):
-options.versions = []
-
-  if (len(args) > 2):
-options.versions.append(args[2])
-
-  if (len(options.versions) <= 0):
 parser.error("At least one version needs to be supplied")
 
   proxy = urllib2.ProxyHandler()
@@ -413,6 +407,8 @@ def main():
   urllib2.install_opener(opener)
 
   projects = options.projects
+  if projects is None:
+parser.error("At least one project needs to be supplied")
 
   if (options.range is True):
 versions = [ Version(v) for v in GetVersions(options.versions, 
projects).getlist() ]



[01/10] hadoop git commit: YARN-3852. Add docker container support to container-executor. Contributed by Abin Shahab.

2015-07-28 Thread aw
Repository: hadoop
Updated Branches:
  refs/heads/HADOOP-12111 4d4f288d3 -> 03335bb4d


YARN-3852. Add docker container support to container-executor. Contributed by 
Abin Shahab.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f36835ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f36835ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f36835ff

Branch: refs/heads/HADOOP-12111
Commit: f36835ff9b878fa20fe58a30f9d1e8c47702d6d2
Parents: 2196e39
Author: Varun Vasudev 
Authored: Mon Jul 27 10:12:30 2015 -0700
Committer: Varun Vasudev 
Committed: Mon Jul 27 10:14:51 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |   3 +
 .../container-executor/impl/configuration.c |  17 +-
 .../container-executor/impl/configuration.h |   2 +
 .../impl/container-executor.c   | 417 ---
 .../impl/container-executor.h   |  25 +-
 .../main/native/container-executor/impl/main.c  |  97 -
 6 files changed, 480 insertions(+), 81 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f36835ff/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3b7d8a8..4e54aea 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -150,6 +150,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. 
 (Jonathan Yaniv and Ishai Menache via curino)
 
+YARN-3852. Add docker container support to container-executor
+(Abin Shahab via vvasudev)
+
   IMPROVEMENTS
 
 YARN-644. Basic null check is not performed on passed in arguments before

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f36835ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index eaa1f19..2825367 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -291,27 +291,23 @@ char ** get_values(const char * key) {
   return extract_values(value);
 }
 
-/**
- * Extracts array of values from the '%' separated list of values.
- */
-char ** extract_values(char *value) {
+char ** extract_values_delim(char *value, const char *delim) {
   char ** toPass = NULL;
   char *tempTok = NULL;
   char *tempstr = NULL;
   int size = 0;
   int toPassSize = MAX_SIZE;
-
   //first allocate any array of 10
   if(value != NULL) {
 toPass = (char **) malloc(sizeof(char *) * toPassSize);
-tempTok = strtok_r((char *)value, "%", &tempstr);
+tempTok = strtok_r((char *)value, delim, &tempstr);
 while (tempTok != NULL) {
   toPass[size++] = tempTok;
   if(size == toPassSize) {
 toPassSize += MAX_SIZE;
 toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize));
   }
-  tempTok = strtok_r(NULL, "%", &tempstr);
+  tempTok = strtok_r(NULL, delim, &tempstr);
 }
   }
   if (toPass != NULL) {
@@ -320,6 +316,13 @@ char ** extract_values(char *value) {
   return toPass;
 }
 
+/**
+ * Extracts array of values from the '%' separated list of values.
+ */
+char ** extract_values(char *value) {
+  extract_values_delim(value, "%");
+}
+
 // free an entry set of values
 void free_values(char** values) {
   if (*values != NULL) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f36835ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
index 133e67b..390a5b5 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/m

[06/10] hadoop git commit: HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)

2015-07-28 Thread aw
HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/030fcfa9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/030fcfa9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/030fcfa9

Branch: refs/heads/HADOOP-12111
Commit: 030fcfa99c345ad57625486eeabedebf2fd4411f
Parents: e21dde5
Author: Arun Suresh 
Authored: Mon Jul 27 23:02:03 2015 -0700
Committer: Arun Suresh 
Committed: Mon Jul 27 23:02:03 2015 -0700

--
 .../apache/hadoop/io/retry/MultiException.java  |  49 +++
 .../hadoop/io/retry/RetryInvocationHandler.java |  99 +-
 hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt |   2 +
 .../ha/ConfiguredFailoverProxyProvider.java |  52 ++-
 .../ha/RequestHedgingProxyProvider.java | 186 ++
 .../markdown/HDFSHighAvailabilityWithNFS.md |   9 +-
 .../markdown/HDFSHighAvailabilityWithQJM.md |  10 +-
 .../ha/TestRequestHedgingProxyProvider.java | 350 +++
 8 files changed, 724 insertions(+), 33 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/030fcfa9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
new file mode 100644
index 000..4963a2d
--- /dev/null
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.io.retry;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Holder class that clients can use to return multiple exceptions.
+ */
+public class MultiException extends IOException {
+
+  private final Map exes;
+
+  public MultiException(Map exes) {
+this.exes = exes;
+  }
+
+  public Map getExceptions() {
+return exes;
+  }
+
+  @Override
+  public String toString() {
+StringBuilder sb = new StringBuilder("{");
+for (Exception e : exes.values()) {
+  sb.append(e.toString()).append(", ");
+}
+sb.append("}");
+return "MultiException[" + sb.toString() + "]";
+  }
+}

http://git-wip-us.apache.org/repos/asf/hadoop/blob/030fcfa9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
--
diff --git 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
index 543567e..9256356 100644
--- 
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
+++ 
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
@@ -23,6 +23,8 @@ import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
 import java.lang.reflect.Proxy;
 import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Map;
 
 import org.apache.commons.logging.Log;
@@ -101,7 +103,7 @@ public class RetryInvocationHandler implements 
RpcInvocationHandler {
 Object ret = invokeMethod(method, args);
 hasMadeASuccessfulCall = true;
 return ret;
-  } catch (Exception e) {
+  } catch (Exception ex) {
 boolean isIdempotentOrAtMostOnce = proxyProvider.getInterface()
 .getMethod(method.getName(), method.getParameterTypes())
 .isAnnotationPresent(Idempotent.class);
@@ -110,15 +112,16 @@ public class RetryInvocationHandler implements 
RpcInvocationHandler {
   .getMethod(method.getName(), method.getParameterTypes())
   .

[05/10] hadoop git commit: HADOOP-12245. References to misspelled REMAINING_QUATA in FileSystemShell.md. Contributed by Gabor Liptak.

2015-07-28 Thread aw
HADOOP-12245. References to misspelled REMAINING_QUATA in FileSystemShell.md. 
Contributed by Gabor Liptak.


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e21dde50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e21dde50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e21dde50

Branch: refs/heads/HADOOP-12111
Commit: e21dde501aa9323b7f34b4bc4ba9d282ec4f2707
Parents: 3572ebd
Author: Akira Ajisaka 
Authored: Tue Jul 28 11:33:10 2015 +0900
Committer: Akira Ajisaka 
Committed: Tue Jul 28 11:33:10 2015 +0900

--
 hadoop-common-project/hadoop-common/CHANGES.txt   | 3 +++
 .../hadoop-common/src/site/markdown/FileSystemShell.md| 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/e21dde50/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt 
b/hadoop-common-project/hadoop-common/CHANGES.txt
index baf39e3..aeaa5b9 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1017,6 +1017,9 @@ Release 2.8.0 - UNRELEASED
 HADOOP-12239. StorageException complaining " no lease ID" when updating
 FolderLastModifiedTime in WASB. (Duo Xu via cnauroth)
 
+HADOOP-12245. References to misspelled REMAINING_QUATA in
+FileSystemShell.md. (Gabor Liptak via aajisaka)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/e21dde50/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
--
diff --git 
a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md 
b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index 144cb73..fb89ca1 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -174,7 +174,7 @@ Usage: `hadoop fs -count [-q] [-h] [-v]  `
 
 Count the number of directories, files and bytes under the paths that match 
the specified file pattern. The output columns with -count are: DIR\_COUNT, 
FILE\_COUNT, CONTENT\_SIZE, PATHNAME
 
-The output columns with -count -q are: QUOTA, REMAINING\_QUATA, SPACE\_QUOTA, 
REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
+The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, 
REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
 
 The -h option shows sizes in human readable format.
 



[07/10] hadoop git commit: YARN-3982. container-executor parsing of container-executor.cfg broken in trunk and branch-2. Contributed by Varun Vasudev

2015-07-28 Thread aw
YARN-3982. container-executor parsing of container-executor.cfg broken
in trunk and branch-2. Contributed by Varun Vasudev


Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f1709342
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f1709342
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f1709342

Branch: refs/heads/HADOOP-12111
Commit: f17093421521efcbdc813f6f2b8411e45ecc7863
Parents: 030fcfa
Author: Xuan 
Authored: Mon Jul 27 23:45:58 2015 -0700
Committer: Xuan 
Committed: Mon Jul 27 23:45:58 2015 -0700

--
 hadoop-yarn-project/CHANGES.txt |  3 +++
 .../container-executor/impl/configuration.c |  4 ++--
 .../test/test-container-executor.c  | 22 +---
 3 files changed, 20 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4f8484a..b4666e8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -698,6 +698,9 @@ Release 2.8.0 - UNRELEASED
 YARN-3846. RM Web UI queue filter is not working for sub queue.
 (Mohammad Shahid Khan via jianhe)
 
+YARN-3982. container-executor parsing of container-executor.cfg broken in
+trunk and branch-2. (Varun Vasudev via xgong)
+
 Release 2.7.2 - UNRELEASED
 
   INCOMPATIBLE CHANGES

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index 2825367..373dbfd 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -284,11 +284,11 @@ char * get_value(const char* key) {
 
 /**
  * Function to return an array of values for a key.
- * Value delimiter is assumed to be a '%'.
+ * Value delimiter is assumed to be a ','.
  */
 char ** get_values(const char * key) {
   char *value = get_value(key);
-  return extract_values(value);
+  return extract_values_delim(value, ",");
 }
 
 char ** extract_values_delim(char *value, const char *delim) {

http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
--
diff --git 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 99bcf34..001a37d 100644
--- 
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ 
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -145,7 +145,7 @@ void check_pid_file(const char* pid_file, pid_t mypid) {
   }
 
   char myPidBuf[33];
-  snprintf(myPidBuf, 33, "%" PRId64, (int64_t)mypid);
+  snprintf(myPidBuf, 33, "%" PRId64, (int64_t)(mypid + 1));
   if (strncmp(pidBuf, myPidBuf, strlen(myPidBuf)) != 0) {
 printf("FAIL: failed to find matching pid in pid file\n");
 printf("FAIL: Expected pid %" PRId64 " : Got %.*s", (int64_t)mypid,
@@ -212,15 +212,15 @@ void test_get_app_log_dir() {
   free(logdir);
 }
 
-void test_check_user() {
+void test_check_user(int expectedFailure) {
   printf("\nTesting test_check_user\n");
   struct passwd *user = check_user(username);
-  if (user == NULL) {
+  if (user == NULL && !expectedFailure) {
 printf("FAIL: failed check for user %s\n", username);
 exit(1);
   }
   free(user);
-  if (check_user("lp") != NULL) {
+  if (check_user("lp") != NULL && !expectedFailure) {
 printf("FAIL: failed check for system user lp\n");
 exit(1);
   }
@@ -228,7 +228,7 @@ void test_check_user() {
 printf("FAIL: failed check for system user roo