hadoop git commit: HDFS-8834. TestReplication is not valid after HDFS-6482. (Contributed by Lei Xu)
Repository: hadoop
Updated Branches:
refs/heads/trunk 975e138df -> f4f1b8b26
HDFS-8834. TestReplication is not valid after HDFS-6482. (Contributed by Lei Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4f1b8b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4f1b8b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4f1b8b2
Branch: refs/heads/trunk
Commit: f4f1b8b267703b8bebab06e17e69a4a4de611592
Parents: 975e138
Author: Lei Xu
Authored: Tue Jul 28 16:55:51 2015 -0700
Committer: Lei Xu
Committed: Tue Jul 28 23:39:26 2015 -0700
--
.../org/apache/hadoop/hdfs/TestReplication.java | 26 +++-
1 file changed, 25 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4f1b8b2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
index b702da0..2139df9 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -25,6 +27,10 @@ import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.InetSocketAddress;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
@@ -505,12 +511,28 @@ public class TestReplication {
if (data_dir.listFiles().length == 0) {
nonParticipatedNodeDirs.add(data_dir);
} else {
+ assertNull("participatedNodeDirs has already been set.",
+ participatedNodeDirs);
participatedNodeDirs = data_dir;
}
}
+ assertEquals(2, nonParticipatedNodeDirs.size());
String blockFile = null;
- File[] listFiles = participatedNodeDirs.listFiles();
+ final List listFiles = new ArrayList<>();
+ Files.walkFileTree(participatedNodeDirs.toPath(),
+ new SimpleFileVisitor() {
+@Override
+public FileVisitResult visitFile(
+java.nio.file.Path file, BasicFileAttributes attrs)
+throws IOException {
+ listFiles.add(file.toFile());
+ return FileVisitResult.CONTINUE;
+}
+ }
+ );
+ assertFalse(listFiles.isEmpty());
+ int numReplicaCreated = 0;
for (File file : listFiles) {
if (file.getName().startsWith(Block.BLOCK_FILE_PREFIX)
&& !file.getName().endsWith("meta")) {
@@ -519,10 +541,12 @@ public class TestReplication {
file1.mkdirs();
new File(file1, blockFile).createNewFile();
new File(file1, blockFile + "_1000.meta").createNewFile();
+numReplicaCreated++;
}
break;
}
}
+ assertEquals(2, numReplicaCreated);
fs.setReplication(new Path("/test"), (short) 3);
cluster.restartDataNodes(); // Lets detect all DNs about dummy copied
hadoop git commit: HDFS-8834. TestReplication is not valid after HDFS-6482. (Contributed by Lei Xu)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 dcf1f1fdf -> 9d8191604
HDFS-8834. TestReplication is not valid after HDFS-6482. (Contributed by Lei Xu)
(cherry picked from commit f4f1b8b267703b8bebab06e17e69a4a4de611592)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d819160
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d819160
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d819160
Branch: refs/heads/branch-2
Commit: 9d8191604010c15a2b9c1eec6e09e04821e3fcee
Parents: dcf1f1f
Author: Lei Xu
Authored: Tue Jul 28 16:55:51 2015 -0700
Committer: Lei Xu
Committed: Tue Jul 28 23:41:44 2015 -0700
--
.../org/apache/hadoop/hdfs/TestReplication.java | 26 +++-
1 file changed, 25 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d819160/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
index 010a26e..55e9056 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReplication.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -25,6 +27,10 @@ import java.io.IOException;
import java.io.OutputStream;
import java.io.RandomAccessFile;
import java.net.InetSocketAddress;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.Random;
@@ -514,12 +520,28 @@ public class TestReplication {
if (data_dir.listFiles().length == 0) {
nonParticipatedNodeDirs.add(data_dir);
} else {
+ assertNull("participatedNodeDirs has already been set.",
+ participatedNodeDirs);
participatedNodeDirs = data_dir;
}
}
+ assertEquals(2, nonParticipatedNodeDirs.size());
String blockFile = null;
- File[] listFiles = participatedNodeDirs.listFiles();
+ final List listFiles = new ArrayList<>();
+ Files.walkFileTree(participatedNodeDirs.toPath(),
+ new SimpleFileVisitor() {
+@Override
+public FileVisitResult visitFile(
+java.nio.file.Path file, BasicFileAttributes attrs)
+throws IOException {
+ listFiles.add(file.toFile());
+ return FileVisitResult.CONTINUE;
+}
+ }
+ );
+ assertFalse(listFiles.isEmpty());
+ int numReplicaCreated = 0;
for (File file : listFiles) {
if (file.getName().startsWith(Block.BLOCK_FILE_PREFIX)
&& !file.getName().endsWith("meta")) {
@@ -528,10 +550,12 @@ public class TestReplication {
file1.mkdirs();
new File(file1, blockFile).createNewFile();
new File(file1, blockFile + "_1000.meta").createNewFile();
+numReplicaCreated++;
}
break;
}
}
+ assertEquals(2, numReplicaCreated);
fs.setReplication(new Path("/test"), (short) 3);
cluster.restartDataNodes(); // Lets detect all DNs about dummy copied
[2/2] hadoop git commit: HDFS-8822. Add SSD storagepolicy tests in TestBlockStoragePolicy#testDefaultPolicies (Contributed by Vinayakumar B)
HDFS-8822. Add SSD storagepolicy tests in
TestBlockStoragePolicy#testDefaultPolicies (Contributed by Vinayakumar B)
(cherry picked from commit 975e138df316f59e8bb0642e138d4b1170fb8184)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dcf1f1fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dcf1f1fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dcf1f1fd
Branch: refs/heads/branch-2
Commit: dcf1f1fdf861fff699c97c74c46c82b541214030
Parents: 0c09597
Author: Vinayakumar B
Authored: Wed Jul 29 11:50:48 2015 +0530
Committer: Vinayakumar B
Committed: Wed Jul 29 11:51:18 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/TestBlockStoragePolicy.java | 88
2 files changed, 75 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcf1f1fd/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ce521a3..28c455e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -417,6 +417,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8811. Move BlockStoragePolicy name's constants from
HdfsServerConstants.java to HdfsConstants.java (vinayakumarb)
+HDFS-8822. Add SSD storagepolicy tests in TestBlockStoragePolicy#
+testDefaultPolicies (vinayakumarb)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcf1f1fd/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 5875b13..ce3fce3 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -68,7 +68,11 @@ public class TestBlockStoragePolicy {
static final EnumSet none = EnumSet.noneOf(StorageType.class);
static final EnumSet archive = EnumSet.of(StorageType.ARCHIVE);
static final EnumSet disk = EnumSet.of(StorageType.DISK);
- static final EnumSet both = EnumSet.of(StorageType.DISK,
StorageType.ARCHIVE);
+ static final EnumSet ssd = EnumSet.of(StorageType.SSD);
+ static final EnumSet disk_archive = EnumSet.of(StorageType.DISK,
+ StorageType.ARCHIVE);
+ static final EnumSet all = EnumSet.of(StorageType.SSD,
+ StorageType.DISK, StorageType.ARCHIVE);
static final long FILE_LEN = 1024;
static final short REPLICATION = 3;
@@ -153,8 +157,8 @@ public class TestBlockStoragePolicy {
final List computed =
cold.chooseStorageTypes(replication);
assertStorageType(computed, replication, StorageType.ARCHIVE);
}
- assertCreationFallback(cold, null, null, null);
- assertReplicationFallback(cold, null, null, null);
+ assertCreationFallback(cold, null, null, null, null, null);
+ assertReplicationFallback(cold, null, null, null, null);
}
{ // check Warm policy
@@ -163,8 +167,10 @@ public class TestBlockStoragePolicy {
final List computed =
warm.chooseStorageTypes(replication);
assertStorageType(computed, replication, StorageType.DISK,
StorageType.ARCHIVE);
}
- assertCreationFallback(warm, StorageType.DISK, StorageType.DISK,
StorageType.ARCHIVE);
- assertReplicationFallback(warm, StorageType.DISK, StorageType.DISK,
StorageType.ARCHIVE);
+ assertCreationFallback(warm, StorageType.DISK, StorageType.DISK,
+ StorageType.ARCHIVE, StorageType.DISK, null);
+ assertReplicationFallback(warm, StorageType.DISK, StorageType.DISK,
+ StorageType.ARCHIVE, StorageType.DISK);
}
{ // check Hot policy
@@ -173,8 +179,50 @@ public class TestBlockStoragePolicy {
final List computed = hot.chooseStorageTypes(replication);
assertStorageType(computed, replication, StorageType.DISK);
}
- assertCreationFallback(hot, null, null, null);
- assertReplicationFallback(hot, StorageType.ARCHIVE, null,
StorageType.ARCHIVE);
+ assertCreationFallback(hot, null, null, null, null, null);
+ assertReplicationFallback(hot, StorageType.ARCHIVE, null,
+ StorageType.ARCHIVE, StorageType.ARCHIVE);
+}
+
+{ // check ONE_SSD policy
+ final BlockStoragePolicy onessd = POLICY_SUITE.getPolicy(ONESSD);
[1/2] hadoop git commit: HDFS-8822. Add SSD storagepolicy tests in TestBlockStoragePolicy#testDefaultPolicies (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 0c0959794 -> dcf1f1fdf
refs/heads/trunk 50887e5b0 -> 975e138df
HDFS-8822. Add SSD storagepolicy tests in
TestBlockStoragePolicy#testDefaultPolicies (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/975e138d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/975e138d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/975e138d
Branch: refs/heads/trunk
Commit: 975e138df316f59e8bb0642e138d4b1170fb8184
Parents: 50887e5
Author: Vinayakumar B
Authored: Wed Jul 29 11:50:48 2015 +0530
Committer: Vinayakumar B
Committed: Wed Jul 29 11:50:48 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/TestBlockStoragePolicy.java | 88
2 files changed, 75 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/975e138d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 654c40f..d30c258 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -760,6 +760,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8811. Move BlockStoragePolicy name's constants from
HdfsServerConstants.java to HdfsConstants.java (vinayakumarb)
+HDFS-8822. Add SSD storagepolicy tests in TestBlockStoragePolicy#
+testDefaultPolicies (vinayakumarb)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/975e138d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index 78e1e58..631d9f7 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -67,7 +67,11 @@ public class TestBlockStoragePolicy {
static final EnumSet none = EnumSet.noneOf(StorageType.class);
static final EnumSet archive = EnumSet.of(StorageType.ARCHIVE);
static final EnumSet disk = EnumSet.of(StorageType.DISK);
- static final EnumSet both = EnumSet.of(StorageType.DISK,
StorageType.ARCHIVE);
+ static final EnumSet ssd = EnumSet.of(StorageType.SSD);
+ static final EnumSet disk_archive = EnumSet.of(StorageType.DISK,
+ StorageType.ARCHIVE);
+ static final EnumSet all = EnumSet.of(StorageType.SSD,
+ StorageType.DISK, StorageType.ARCHIVE);
static final long FILE_LEN = 1024;
static final short REPLICATION = 3;
@@ -153,8 +157,8 @@ public class TestBlockStoragePolicy {
final List computed =
cold.chooseStorageTypes(replication);
assertStorageType(computed, replication, StorageType.ARCHIVE);
}
- assertCreationFallback(cold, null, null, null);
- assertReplicationFallback(cold, null, null, null);
+ assertCreationFallback(cold, null, null, null, null, null);
+ assertReplicationFallback(cold, null, null, null, null);
}
{ // check Warm policy
@@ -163,8 +167,10 @@ public class TestBlockStoragePolicy {
final List computed =
warm.chooseStorageTypes(replication);
assertStorageType(computed, replication, StorageType.DISK,
StorageType.ARCHIVE);
}
- assertCreationFallback(warm, StorageType.DISK, StorageType.DISK,
StorageType.ARCHIVE);
- assertReplicationFallback(warm, StorageType.DISK, StorageType.DISK,
StorageType.ARCHIVE);
+ assertCreationFallback(warm, StorageType.DISK, StorageType.DISK,
+ StorageType.ARCHIVE, StorageType.DISK, null);
+ assertReplicationFallback(warm, StorageType.DISK, StorageType.DISK,
+ StorageType.ARCHIVE, StorageType.DISK);
}
{ // check Hot policy
@@ -173,8 +179,50 @@ public class TestBlockStoragePolicy {
final List computed = hot.chooseStorageTypes(replication);
assertStorageType(computed, replication, StorageType.DISK);
}
- assertCreationFallback(hot, null, null, null);
- assertReplicationFallback(hot, StorageType.ARCHIVE, null,
StorageType.ARCHIVE);
+ assertCreationFallback(hot, null, null, null, null, null);
+ assertReplicationFallback(hot, StorageType.ARCHIVE, null,
+ StorageType.ARCHIVE, StorageType.ARCHIVE);
+}
+
+{ // check ONE_SSD policy
+ final BlockSt
[2/2] hadoop git commit: HDFS-8811. Move BlockStoragePolicy name's constants from HdfsServerConstants.java to HdfsConstants.java (Contributed by Vinayakumar B)
HDFS-8811. Move BlockStoragePolicy name's constants from
HdfsServerConstants.java to HdfsConstants.java (Contributed by Vinayakumar B)
(cherry picked from commit 50887e5b07b6abb20c0edd74211e5612dc7b16da)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c095979
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c095979
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c095979
Branch: refs/heads/branch-2
Commit: 0c09597942284d99db729d9a9eb4df682886078b
Parents: 71a5aa2
Author: Vinayakumar B
Authored: Wed Jul 29 11:45:33 2015 +0530
Committer: Vinayakumar B
Committed: Wed Jul 29 11:46:19 2015 +0530
--
.../hadoop/hdfs/protocol/HdfsConstants.java | 3 ++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../BlockStoragePolicySuite.java| 6 ++--
.../hdfs/server/common/HdfsServerConstants.java | 3 --
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 2 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 30 ++--
.../hdfs/server/mover/TestStorageMover.java | 8 +++---
7 files changed, 29 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c095979/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index f12a6fb..d4fb860 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -37,6 +37,9 @@ public class HdfsConstants {
public static final String MEMORY_STORAGE_POLICY_NAME = "LAZY_PERSIST";
public static final String ALLSSD_STORAGE_POLICY_NAME = "ALL_SSD";
public static final String ONESSD_STORAGE_POLICY_NAME = "ONE_SSD";
+ public static final String HOT_STORAGE_POLICY_NAME = "HOT";
+ public static final String WARM_STORAGE_POLICY_NAME = "WARM";
+ public static final String COLD_STORAGE_POLICY_NAME = "COLD";
// TODO should be conf injected?
public static final int DEFAULT_DATA_SOCKET_SIZE = 128 * 1024;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c095979/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1efea0f..ce521a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -414,6 +414,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8180. AbstractFileSystem Implementation for WebHdfs. (snayak via
jghoman)
+HDFS-8811. Move BlockStoragePolicy name's constants from
+HdfsServerConstants.java to HdfsConstants.java (vinayakumarb)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c095979/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
index 2a71c29..7e6c2b2 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
@@ -69,18 +69,18 @@ public class BlockStoragePolicySuite {
new StorageType[]{StorageType.SSD, StorageType.DISK});
final byte hotId = HdfsServerConstants.HOT_STORAGE_POLICY_ID;
policies[hotId] = new BlockStoragePolicy(hotId,
-HdfsServerConstants.HOT_STORAGE_POLICY_NAME,
+HdfsConstants.HOT_STORAGE_POLICY_NAME,
new StorageType[]{StorageType.DISK}, StorageType.EMPTY_ARRAY,
new StorageType[]{StorageType.ARCHIVE});
final byte warmId = HdfsServerConstants.WARM_STORAGE_POLICY_ID;
policies[warmId] = new BlockStoragePolicy(warmId,
-HdfsServerConstants.WARM_STORAGE_POLICY_NAME,
+HdfsConstants.WARM_STORAGE_POLICY_NAME,
new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
new
[1/2] hadoop git commit: HDFS-8811. Move BlockStoragePolicy name's constants from HdfsServerConstants.java to HdfsConstants.java (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 71a5aa267 -> 0c0959794
refs/heads/trunk 0712a8103 -> 50887e5b0
HDFS-8811. Move BlockStoragePolicy name's constants from
HdfsServerConstants.java to HdfsConstants.java (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/50887e5b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/50887e5b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/50887e5b
Branch: refs/heads/trunk
Commit: 50887e5b07b6abb20c0edd74211e5612dc7b16da
Parents: 0712a81
Author: Vinayakumar B
Authored: Wed Jul 29 11:45:33 2015 +0530
Committer: Vinayakumar B
Committed: Wed Jul 29 11:45:33 2015 +0530
--
.../hadoop/hdfs/protocol/HdfsConstants.java | 3 ++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../BlockStoragePolicySuite.java| 6 ++--
.../hdfs/server/common/HdfsServerConstants.java | 3 --
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 2 +-
.../hadoop/hdfs/TestBlockStoragePolicy.java | 30 ++--
.../hdfs/server/mover/TestStorageMover.java | 8 +++---
7 files changed, 29 insertions(+), 26 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50887e5b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 58c7ea1..443576d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -37,6 +37,9 @@ public final class HdfsConstants {
public static final String MEMORY_STORAGE_POLICY_NAME = "LAZY_PERSIST";
public static final String ALLSSD_STORAGE_POLICY_NAME = "ALL_SSD";
public static final String ONESSD_STORAGE_POLICY_NAME = "ONE_SSD";
+ public static final String HOT_STORAGE_POLICY_NAME = "HOT";
+ public static final String WARM_STORAGE_POLICY_NAME = "WARM";
+ public static final String COLD_STORAGE_POLICY_NAME = "COLD";
// TODO should be conf injected?
public static final int DEFAULT_DATA_SOCKET_SIZE = 128 * 1024;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50887e5b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ef12720..654c40f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -757,6 +757,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8180. AbstractFileSystem Implementation for WebHdfs. (snayak via
jghoman)
+HDFS-8811. Move BlockStoragePolicy name's constants from
+HdfsServerConstants.java to HdfsConstants.java (vinayakumarb)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/50887e5b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
index 2a71c29..7e6c2b2 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStoragePolicySuite.java
@@ -69,18 +69,18 @@ public class BlockStoragePolicySuite {
new StorageType[]{StorageType.SSD, StorageType.DISK});
final byte hotId = HdfsServerConstants.HOT_STORAGE_POLICY_ID;
policies[hotId] = new BlockStoragePolicy(hotId,
-HdfsServerConstants.HOT_STORAGE_POLICY_NAME,
+HdfsConstants.HOT_STORAGE_POLICY_NAME,
new StorageType[]{StorageType.DISK}, StorageType.EMPTY_ARRAY,
new StorageType[]{StorageType.ARCHIVE});
final byte warmId = HdfsServerConstants.WARM_STORAGE_POLICY_ID;
policies[warmId] = new BlockStoragePolicy(warmId,
-HdfsServerConstants.WARM_STORAGE_POLICY_NAME,
+HdfsConstants.WARM_STORAGE_POLICY_NAME,
new StorageType[]{StorageType.DISK, StorageType.ARCHIVE},
new StorageT
hadoop git commit: HDFS-8180. AbstractFileSystem Implementation for WebHdfs. Contributed by Sathosh G Nayak.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 587593566 -> 71a5aa267
HDFS-8180. AbstractFileSystem Implementation for WebHdfs. Contributed by
Sathosh G Nayak.
(cherry picked from commit 0712a8103fec6e9a9ceba335e3c3800b85b2c7ca)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71a5aa26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71a5aa26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71a5aa26
Branch: refs/heads/branch-2
Commit: 71a5aa267d8e93308f2546002973cefc89c8a8f8
Parents: 5875935
Author: Jakob Homan
Authored: Tue Jul 28 21:03:31 2015 -0700
Committer: Jakob Homan
Committed: Tue Jul 28 21:11:02 2015 -0700
--
.../src/main/resources/core-default.xml | 12 ++
.../fs/FileContextMainOperationsBaseTest.java | 4 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../java/org/apache/hadoop/fs/SWebHdfs.java | 51 ++
.../main/java/org/apache/hadoop/fs/WebHdfs.java | 51 ++
.../main/java/org/apache/hadoop/fs/package.html | 26 +++
.../TestSWebHdfsFileContextMainOperations.java | 110 +
.../TestWebHdfsFileContextMainOperations.java | 157 +++
8 files changed, 411 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a5aa26/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 8643a0e..ee42675 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -597,6 +597,18 @@ for ldap providers in the same way as above does.
+ fs.AbstractFileSystem.webhdfs.impl
+ org.apache.hadoop.fs.WebHdfs
+ The FileSystem for webhdfs: uris.
+
+
+
+ fs.AbstractFileSystem.swebhdfs.impl
+ org.apache.hadoop.fs.SWebHdfs
+ The FileSystem for swebhdfs: uris.
+
+
+
fs.ftp.host
0.0.0.0
FTP filesystem connects to this server
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a5aa26/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
index e872176..12ec375 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
@@ -1249,7 +1249,7 @@ public abstract class FileContextMainOperationsBaseTest {
byte[] bb = new byte[(int)len];
FSDataInputStream fsdis = fc.open(path);
try {
- fsdis.read(bb);
+ fsdis.readFully(bb);
} finally {
fsdis.close();
}
@@ -1310,7 +1310,7 @@ public abstract class FileContextMainOperationsBaseTest {
byte[] bb = new byte[data.length];
FSDataInputStream fsdis = fc.open(path);
try {
- fsdis.read(bb);
+ fsdis.readFully(bb);
} finally {
fsdis.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a5aa26/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9dc2a82..1efea0f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -412,6 +412,8 @@ Release 2.8.0 - UNRELEASED
HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)
+HDFS-8180. AbstractFileSystem Implementation for WebHdfs. (snayak via
jghoman)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71a5aa26/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
new file mode 100644
index 000..cd36393
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more con
hadoop git commit: HDFS-8180. AbstractFileSystem Implementation for WebHdfs. Contributed by Sathosh G Nayak.
Repository: hadoop
Updated Branches:
refs/heads/trunk 69b095730 -> 0712a8103
HDFS-8180. AbstractFileSystem Implementation for WebHdfs. Contributed by
Sathosh G Nayak.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0712a810
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0712a810
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0712a810
Branch: refs/heads/trunk
Commit: 0712a8103fec6e9a9ceba335e3c3800b85b2c7ca
Parents: 69b0957
Author: Jakob Homan
Authored: Tue Jul 28 21:03:31 2015 -0700
Committer: Jakob Homan
Committed: Tue Jul 28 21:03:31 2015 -0700
--
.../src/main/resources/core-default.xml | 12 ++
.../fs/FileContextMainOperationsBaseTest.java | 4 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../java/org/apache/hadoop/fs/SWebHdfs.java | 51 ++
.../main/java/org/apache/hadoop/fs/WebHdfs.java | 51 ++
.../main/java/org/apache/hadoop/fs/package.html | 26 +++
.../TestSWebHdfsFileContextMainOperations.java | 110 +
.../TestWebHdfsFileContextMainOperations.java | 157 +++
8 files changed, 411 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0712a810/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 13702ee..bfdd453 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -596,6 +596,18 @@ for ldap providers in the same way as above does.
+ fs.AbstractFileSystem.webhdfs.impl
+ org.apache.hadoop.fs.WebHdfs
+ The FileSystem for webhdfs: uris.
+
+
+
+ fs.AbstractFileSystem.swebhdfs.impl
+ org.apache.hadoop.fs.SWebHdfs
+ The FileSystem for swebhdfs: uris.
+
+
+
fs.ftp.host
0.0.0.0
FTP filesystem connects to this server
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0712a810/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
index e872176..12ec375 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileContextMainOperationsBaseTest.java
@@ -1249,7 +1249,7 @@ public abstract class FileContextMainOperationsBaseTest {
byte[] bb = new byte[(int)len];
FSDataInputStream fsdis = fc.open(path);
try {
- fsdis.read(bb);
+ fsdis.readFully(bb);
} finally {
fsdis.close();
}
@@ -1310,7 +1310,7 @@ public abstract class FileContextMainOperationsBaseTest {
byte[] bb = new byte[data.length];
FSDataInputStream fsdis = fc.open(path);
try {
- fsdis.read(bb);
+ fsdis.readFully(bb);
} finally {
fsdis.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0712a810/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9b2de81..ef12720 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -755,6 +755,8 @@ Release 2.8.0 - UNRELEASED
HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)
+HDFS-8180. AbstractFileSystem Implementation for WebHdfs. (snayak via
jghoman)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0712a810/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
new file mode 100644
index 000..cd36393
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/fs/SWebHdfs.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this
[07/50] [abbrv] hadoop git commit: YARN-3941. Proportional Preemption policy should try to avoid sending duplicate PREEMPT_CONTAINER event to scheduler. (Sunil G via wangda)
YARN-3941. Proportional Preemption policy should try to avoid sending duplicate
PREEMPT_CONTAINER event to scheduler. (Sunil G via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3bba1800
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3bba1800
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3bba1800
Branch: refs/heads/YARN-1197
Commit: 3bba1800513b38a4827f7552f348db87dc47c783
Parents: ee98d63
Author: Wangda Tan
Authored: Thu Jul 23 10:07:57 2015 -0700
Committer: Wangda Tan
Committed: Thu Jul 23 10:07:57 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 2 ++
.../capacity/ProportionalCapacityPreemptionPolicy.java | 9 ++---
.../capacity/TestProportionalCapacityPreemptionPolicy.java | 6 +++---
3 files changed, 11 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bba1800/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 93962f1..9416cd6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -664,6 +664,8 @@ Release 2.8.0 - UNRELEASED
YARN-3956. Fix TestNodeManagerHardwareUtils fails on Mac (Varun Vasudev
via wangda)
+YARN-3941. Proportional Preemption policy should try to avoid sending
duplicate PREEMPT_CONTAINER event to scheduler. (Sunil G via wangda)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bba1800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
index 1152cef..77df059 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java
@@ -260,13 +260,16 @@ public class ProportionalCapacityPreemptionPolicy
implements SchedulingEditPolic
SchedulerEventType.KILL_CONTAINER));
preempted.remove(container);
} else {
+ if (preempted.get(container) != null) {
+// We already updated the information to scheduler earlier, we need
+// not have to raise another event.
+continue;
+ }
//otherwise just send preemption events
rmContext.getDispatcher().getEventHandler().handle(
new ContainerPreemptEvent(appAttemptId, container,
SchedulerEventType.PREEMPT_CONTAINER));
- if (preempted.get(container) == null) {
-preempted.put(container, clock.getTime());
- }
+ preempted.put(container, clock.getTime());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bba1800/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
index bc4d0dc..8d9f48a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/mo
[03/50] [abbrv] hadoop git commit: YARN-3954. Fix TestYarnConfigurationFields#testCompareConfigurationClassAgainstXml. (varun saxena via rohithsharmaks)
YARN-3954. Fix TestYarnConfigurationFields#testCompareConfigurationClassAgainstXml. (varun saxena via rohithsharmaks) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8376ea32 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8376ea32 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8376ea32 Branch: refs/heads/YARN-1197 Commit: 8376ea3297a3eab33df27454b18cf215cfb7c6ff Parents: 76ec26d Author: rohithsharmaks Authored: Thu Jul 23 00:28:24 2015 +0530 Committer: rohithsharmaks Committed: Thu Jul 23 00:28:24 2015 +0530 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../src/main/resources/yarn-default.xml | 10 ++ 2 files changed, 13 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8376ea32/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f751862..eb52745 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -656,6 +656,9 @@ Release 2.8.0 - UNRELEASED YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo should based on total-used-resources. (Bibin A Chundatt via wangda) +YARN-3954. Fix TestYarnConfigurationFields#testCompareConfigurationClassAgainstXml. +(varun saxena via rohithsharmaks) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/8376ea32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 2edeef0..d586f51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2131,4 +2131,14 @@ false + + +Defines maximum application priority in a cluster. +If an application is submitted with a priority higher than this value, it will be +reset to this maximum value. + +yarn.cluster.max-application-priority +0 + +
[40/50] [abbrv] hadoop git commit: YARN-3853. Add docker container runtime support to LinuxContainterExecutor. Contributed by Sidharta Seethana.
YARN-3853. Add docker container runtime support to LinuxContainterExecutor. Contributed by Sidharta Seethana. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e6fce91 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e6fce91 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e6fce91 Branch: refs/heads/YARN-1197 Commit: 3e6fce91a471b4a5099de109582e7c6417e8a822 Parents: f36835f Author: Varun Vasudev Authored: Mon Jul 27 11:57:40 2015 -0700 Committer: Varun Vasudev Committed: Mon Jul 27 11:57:40 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 4 + .../server/nodemanager/ContainerExecutor.java | 23 +- .../nodemanager/DefaultContainerExecutor.java | 2 +- .../nodemanager/DockerContainerExecutor.java| 2 +- .../nodemanager/LinuxContainerExecutor.java | 222 +++ .../launcher/ContainerLaunch.java | 15 + .../linux/privileged/PrivilegedOperation.java | 46 +++- .../PrivilegedOperationException.java | 30 +- .../privileged/PrivilegedOperationExecutor.java | 30 +- .../linux/resources/CGroupsHandler.java | 8 + .../linux/resources/CGroupsHandlerImpl.java | 12 +- .../runtime/DefaultLinuxContainerRuntime.java | 148 ++ .../DelegatingLinuxContainerRuntime.java| 110 .../runtime/DockerLinuxContainerRuntime.java| 273 +++ .../linux/runtime/LinuxContainerRuntime.java| 38 +++ .../runtime/LinuxContainerRuntimeConstants.java | 69 + .../linux/runtime/docker/DockerClient.java | 82 ++ .../linux/runtime/docker/DockerCommand.java | 66 + .../linux/runtime/docker/DockerLoadCommand.java | 30 ++ .../linux/runtime/docker/DockerRunCommand.java | 107 .../runtime/ContainerExecutionException.java| 85 ++ .../runtime/ContainerRuntime.java | 50 .../runtime/ContainerRuntimeConstants.java | 33 +++ .../runtime/ContainerRuntimeContext.java| 105 +++ .../executor/ContainerLivenessContext.java | 13 + .../executor/ContainerReacquisitionContext.java | 13 + .../executor/ContainerSignalContext.java| 13 + .../executor/ContainerStartContext.java | 23 +- .../TestLinuxContainerExecutorWithMocks.java| 118 +--- .../TestPrivilegedOperationExecutor.java| 8 +- .../runtime/TestDockerContainerRuntime.java | 219 +++ 31 files changed, 1815 insertions(+), 182 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4e54aea..534c55a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -153,6 +153,10 @@ Release 2.8.0 - UNRELEASED YARN-3852. Add docker container support to container-executor (Abin Shahab via vvasudev) +YARN-3853. Add docker container runtime support to LinuxContainterExecutor. +(Sidharta Seethana via vvasudev) + + IMPROVEMENTS YARN-644. Basic null check is not performed on passed in arguments before http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 79f9b0d..68bfbbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -24,8 +24,10 @@ import java.io.OutputStream; import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -39,6 +41,7 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apac
[27/50] [abbrv] hadoop git commit: HADOOP-11807. add a lint mode to releasedocmaker (ramtin via aw)
HADOOP-11807. add a lint mode to releasedocmaker (ramtin via aw)
(cherry picked from commit 8e657fba2fd33f7550597ea9c4c6e9a87aa1ef1c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/098ba450
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/098ba450
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/098ba450
Branch: refs/heads/YARN-1197
Commit: 098ba450cc98475b84d60bb5ac3bd7b558b2a67c
Parents: a3bd7b4
Author: Allen Wittenauer
Authored: Sat Jun 27 08:59:50 2015 -0700
Committer: Andrew Wang
Committed: Fri Jul 24 18:31:23 2015 -0700
--
dev-support/releasedocmaker.py | 76 +---
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
2 files changed, 68 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/098ba450/dev-support/releasedocmaker.py
--
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index 2ccc1c0..8e68b3c 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -87,8 +87,15 @@ def notableclean(str):
str=str.rstrip()
return str
+# clean output dir
+def cleanOutputDir(dir):
+files = os.listdir(dir)
+for name in files:
+os.remove(os.path.join(dir,name))
+os.rmdir(dir)
+
def mstr(obj):
- if (obj == None):
+ if (obj is None):
return ""
return unicode(obj)
@@ -148,7 +155,7 @@ class Jira:
return mstr(self.fields['description'])
def getReleaseNote(self):
-if (self.notes == None):
+if (self.notes is None):
field = self.parent.fieldIdMap['Release Note']
if (self.fields.has_key(field)):
self.notes=mstr(self.fields[field])
@@ -159,14 +166,14 @@ class Jira:
def getPriority(self):
ret = ""
pri = self.fields['priority']
-if(pri != None):
+if(pri is not None):
ret = pri['name']
return mstr(ret)
def getAssignee(self):
ret = ""
mid = self.fields['assignee']
-if(mid != None):
+if(mid is not None):
ret = mid['displayName']
return mstr(ret)
@@ -182,21 +189,21 @@ class Jira:
def getType(self):
ret = ""
mid = self.fields['issuetype']
-if(mid != None):
+if(mid is not None):
ret = mid['name']
return mstr(ret)
def getReporter(self):
ret = ""
mid = self.fields['reporter']
-if(mid != None):
+if(mid is not None):
ret = mid['displayName']
return mstr(ret)
def getProject(self):
ret = ""
mid = self.fields['project']
-if(mid != None):
+if(mid is not None):
ret = mid['key']
return mstr(ret)
@@ -214,7 +221,7 @@ class Jira:
return False
def getIncompatibleChange(self):
-if (self.incompat == None):
+if (self.incompat is None):
field = self.parent.fieldIdMap['Hadoop Flags']
self.reviewed=False
self.incompat=False
@@ -227,6 +234,24 @@ class Jira:
self.reviewed=True
return self.incompat
+ def checkMissingComponent(self):
+ if (len(self.fields['components'])>0):
+ return False
+ return True
+
+ def checkMissingAssignee(self):
+ if (self.fields['assignee'] is not None):
+ return False
+ return True
+
+ def checkVersionString(self):
+ field = self.parent.fieldIdMap['Fix Version/s']
+ for h in self.fields[field]:
+ found = re.match('^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', h['name'])
+ if not found:
+ return True
+ return False
+
def getReleaseDate(self,version):
for j in range(len(self.fields['fixVersions'])):
if self.fields['fixVersions'][j]==version:
@@ -339,9 +364,11 @@ def main():
help="build an index file")
parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
help="use current date for unreleased versions")
+ parser.add_option("-n","--lint", dest="lint", action="store_true",
+ help="use lint flag to exit on failures")
(options, args) = parser.parse_args()
- if (options.versions == None):
+ if (options.versions is None):
options.versions = []
if (len(args) > 2):
@@ -396,6 +423,9 @@ def main():
reloutputs.writeAll(relhead)
choutputs.writeAll(chhead)
+ errorCount=0
+ warningCount=0
+ lintMessage=""
incompatlist=[]
buglist=[]
improvementlist=[]
@@ -408,6 +438,14 @@ def main():
for jira in sorted(jlist):
if jira.getIncompatibleChange():
incompatlist.append(jira)
+ if (len(jira.getReleaseNote())==0):
+ warningCount+=1
+
+if jira.checkVersionString():
+ warningCount+=1
+
+if jira.checkMissingComponent() or jira.checkMissingAssignee():
+ errorCount+=1
elif jira.getType() =
[47/50] [abbrv] hadoop git commit: YARN-3866. AM-RM protocol changes to support container resizing. Contributed by Meng Ding
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7758afd3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java
deleted file mode 100644
index 29b0ffe..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceDecrease.java
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api;
-
-import org.junit.Assert;
-
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease;
-import org.apache.hadoop.yarn.api.records.Resource;
-import
org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceDecreasePBImpl;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProto;
-import org.junit.Test;
-
-public class TestContainerResourceDecrease {
- @Test
- public void testResourceDecreaseContext() {
-ContainerId containerId = ContainerId
-.newContainerId(ApplicationAttemptId.newInstance(
-ApplicationId.newInstance(1234, 3), 3), 7);
-Resource resource = Resource.newInstance(1023, 3);
-ContainerResourceDecrease ctx = ContainerResourceDecrease.newInstance(
-containerId, resource);
-
-// get proto and recover to ctx
-ContainerResourceDecreaseProto proto =
-((ContainerResourceDecreasePBImpl) ctx).getProto();
-ctx = new ContainerResourceDecreasePBImpl(proto);
-
-// check values
-Assert.assertEquals(ctx.getCapability(), resource);
-Assert.assertEquals(ctx.getContainerId(), containerId);
- }
-
- @Test
- public void testResourceDecreaseContextWithNull() {
-ContainerResourceDecrease ctx = ContainerResourceDecrease.newInstance(null,
-null);
-
-// get proto and recover to ctx;
-ContainerResourceDecreaseProto proto =
-((ContainerResourceDecreasePBImpl) ctx).getProto();
-ctx = new ContainerResourceDecreasePBImpl(proto);
-
-// check values
-Assert.assertNull(ctx.getCapability());
-Assert.assertNull(ctx.getContainerId());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7758afd3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java
deleted file mode 100644
index 932d5a7..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestContainerResourceIncrease.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api;
[01/50] [abbrv] hadoop git commit: HADOOP-12184. Remove unused Linux-specific constants in NativeIO (Martin Walsh via Colin P. McCabe)
Repository: hadoop
Updated Branches:
refs/heads/YARN-1197 2934ff345 -> cb9566269 (forced update)
HADOOP-12184. Remove unused Linux-specific constants in NativeIO (Martin Walsh
via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b3bceb5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b3bceb5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b3bceb5
Branch: refs/heads/YARN-1197
Commit: 1b3bceb58c8e536a75fa3f99cc3ceeaba91a07de
Parents: efa9724
Author: Colin Patrick Mccabe
Authored: Wed Jul 22 11:11:38 2015 -0700
Committer: Colin Patrick Mccabe
Committed: Wed Jul 22 11:34:10 2015 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java| 4
2 files changed, 3 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b3bceb5/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index c0e5c92..ff7d2ad 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -713,6 +713,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12214. Parse 'HadoopArchive' commandline using cli Options.
(vinayakumarb)
+HADOOP-12184. Remove unused Linux-specific constants in NativeIO (Martin
+Walsh via Colin P. McCabe)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b3bceb5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
index 688b955..77a40ea 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/nativeio/NativeIO.java
@@ -67,9 +67,6 @@ public class NativeIO {
public static final int O_APPEND = 02000;
public static final int O_NONBLOCK = 04000;
public static final int O_SYNC = 01;
-public static final int O_ASYNC = 02;
-public static final int O_FSYNC = O_SYNC;
-public static final int O_NDELAY = O_NONBLOCK;
// Flags for posix_fadvise() from bits/fcntl.h
/* No further special treatment. */
@@ -356,7 +353,6 @@ public class NativeIO {
public static final int S_IFREG = 010; /* regular */
public static final int S_IFLNK = 012; /* symbolic link */
public static final int S_IFSOCK = 014; /* socket */
- public static final int S_IFWHT = 016; /* whiteout */
public static final int S_ISUID = 0004000; /* set user id on execution
*/
public static final int S_ISGID = 0002000; /* set group id on execution
*/
public static final int S_ISVTX = 0001000; /* save swapped text even
after use */
[15/50] [abbrv] hadoop git commit: YARN-3845. Scheduler page does not render RGBA color combinations in IE11. (Contributed by Mohammad Shahid Khan)
YARN-3845. Scheduler page does not render RGBA color combinations in IE11.
(Contributed by Mohammad Shahid Khan)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e202efaf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e202efaf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e202efaf
Branch: refs/heads/YARN-1197
Commit: e202efaf932c940e6da5fe857ae55c0808fd4fdd
Parents: 02c0181
Author: Rohith Sharma K S
Authored: Fri Jul 24 12:43:06 2015 +0530
Committer: Rohith Sharma K S
Committed: Fri Jul 24 12:43:06 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../apache/hadoop/yarn/webapp/view/TwoColumnLayout.java | 2 +-
.../resourcemanager/webapp/CapacitySchedulerPage.java | 7 ---
.../resourcemanager/webapp/DefaultSchedulerPage.java | 4 ++--
.../server/resourcemanager/webapp/FairSchedulerPage.java | 10 ++
5 files changed, 16 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e202efaf/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3d41ba7..f23853b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -669,6 +669,9 @@ Release 2.8.0 - UNRELEASED
YARN-3900. Protobuf layout of yarn_security_token causes errors in other
protos
that include it (adhoot via rkanter)
+YARN-3845. Scheduler page does not render RGBA color combinations in IE11.
+(Contributed by Mohammad Shahid Khan)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e202efaf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java
index b8f5f75..4d7752d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/TwoColumnLayout.java
@@ -126,7 +126,7 @@ public class TwoColumnLayout extends HtmlPage {
styles.add(join('#', tableId, "_paginate span {font-weight:normal}"));
styles.add(join('#', tableId, " .progress {width:8em}"));
styles.add(join('#', tableId, "_processing {top:-1.5em; font-size:1em;"));
-styles.add(" color:#000; background:rgba(255, 255, 255, 0.8)}");
+styles.add(" color:#000; background:#fefefe}");
for (String style : innerStyles) {
styles.add(join('#', tableId, " ", style));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e202efaf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index a784601..12a3013 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -59,9 +59,10 @@ class CapacitySchedulerPage extends RmView {
static final float Q_MAX_WIDTH = 0.8f;
static final float Q_STATS_POS = Q_MAX_WIDTH + 0.05f;
static final String Q_END = "left:101%";
- static final String Q_GIVEN = "left:0%;background:none;border:1px dashed
rgba(0,0,0,0.25)";
- static final String Q_OVER = "background:rgba(255, 140, 0, 0.8)";
- static final String Q_UNDER = "background:rgba(50, 205, 50, 0.8)";
+ static final String Q_GIVEN =
+ "left:0%;background:none;border:1px dashed #BFBFBF";
+ static final String Q_OVER = "background:#FFA333";
+ static final String Q_UNDER = "background:#5BD75B";
@RequestScoped
static class CSQInfo {
http://git-wip-us.apache.org/repos/asf
[09/50] [abbrv] hadoop git commit: HDFS-8730. Clean up the import statements in ClientProtocol. Contributed by Takanobu Asanuma.
HDFS-8730. Clean up the import statements in ClientProtocol. Contributed by
Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/813cf89b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/813cf89b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/813cf89b
Branch: refs/heads/YARN-1197
Commit: 813cf89bb56ad1a48b35fd44644d63540e8fa7d1
Parents: adfa34f
Author: Haohui Mai
Authored: Thu Jul 23 10:30:17 2015 -0700
Committer: Haohui Mai
Committed: Thu Jul 23 10:31:11 2015 -0700
--
.../hadoop/hdfs/protocol/ClientProtocol.java| 306 +++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
2 files changed, 182 insertions(+), 127 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/813cf89b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 381be30..713c23c 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hdfs.protocol;
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.EnumSet;
import java.util.List;
@@ -29,14 +28,9 @@ import
org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
import org.apache.hadoop.fs.CacheFlag;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
-import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FsServerDefaults;
-import org.apache.hadoop.fs.InvalidPathException;
import org.apache.hadoop.fs.Options;
-import org.apache.hadoop.fs.Options.Rename;
-import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
@@ -48,14 +42,11 @@ import
org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import
org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSelector;
-import org.apache.hadoop.hdfs.server.namenode.NotReplicatedYetException;
-import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.retry.AtMostOnce;
import org.apache.hadoop.io.retry.Idempotent;
-import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenInfo;
@@ -121,9 +112,12 @@ public interface ClientProtocol {
*
* @return file length and array of blocks with their locations
*
- * @throws AccessControlException If access is denied
- * @throws FileNotFoundException If file src does not exist
- * @throws UnresolvedLinkException If src contains a symlink
+ * @throws org.apache.hadoop.security.AccessControlException If access is
+ * denied
+ * @throws java.io.FileNotFoundException If file src does not
+ * exist
+ * @throws org.apache.hadoop.fs.UnresolvedLinkException If src
+ * contains a symlink
* @throws IOException If an I/O error occurred
*/
@Idempotent
@@ -166,24 +160,29 @@ public interface ClientProtocol {
*
* @return the status of the created file, it could be null if the server
* doesn't support returning the file status
- * @throws AccessControlException If access is denied
+ * @throws org.apache.hadoop.security.AccessControlException If access is
+ * denied
* @throws AlreadyBeingCreatedException if the path does not exist.
* @throws DSQuotaExceededException If file creation violates disk space
* quota restriction
- * @throws FileAlreadyExistsException If file src already exists
- * @throws FileNotFoundException If parent of src does not exist
- * and createParent is false
- * @throws ParentNotDirectoryException If parent of src is not a
- * directory.
+ * @throws org.apache.hadoop.fs.
[23/50] [abbrv] hadoop git commit: YARN-3026. Move application-specific container allocation logic from LeafQueue to FiCaSchedulerApp. Contributed by Wangda Tan
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83fe34ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index dfeb30f..c660fcb 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -24,6 +24,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang.mutable.MutableObject;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -39,6 +40,9 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger;
import
org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEvent;
import
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
@@ -48,11 +52,22 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManage
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityHeadroomProvider;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
+import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+import com.google.common.annotations.VisibleForTesting;
/**
* Represents an application attempt from the viewpoint of the FIFO or Capacity
@@ -61,14 +76,22 @@ import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@Private
@Unstable
public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
-
private static final Log LOG = LogFactory.getLog(FiCaSchedulerApp.class);
+ static final CSAssignment NULL_ASSIGNMENT =
+ new CSAssignment(Resources.createResource(0, 0), NodeType.NODE_LOCAL);
+
+ static final CSAssignment SKIP_ASSIGNMENT = new CSAssignment(true);
+
private final Set containersToPreempt =
new HashSet();
private CapacityHeadroomProvider headroomProvider;
+ private ResourceCalculator rc = new DefaultResourceCalculator();
+
+ private ResourceScheduler scheduler;
+
public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId,
String user, Queue queue, ActiveUsersManager activeUsersManager,
RMContext rmContext) {
@@ -95,6 +118,12 @@ public class FiCaSchedulerApp extends
SchedulerApplicationAttempt {
setAMResource(am
[45/50] [abbrv] hadoop git commit: HADOOP-12175. FsShell must load SpanReceierHost to support tracing (Masatake Iwasaki via Colin P. McCabe)
HADOOP-12175. FsShell must load SpanReceierHost to support tracing (Masatake
Iwasaki via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69b09573
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69b09573
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69b09573
Branch: refs/heads/YARN-1197
Commit: 69b095730bd87706ddcbabeea6c513290ee4cdf5
Parents: f170934
Author: Colin Patrick Mccabe
Authored: Tue Jul 28 12:49:38 2015 -0700
Committer: Colin Patrick Mccabe
Committed: Tue Jul 28 12:56:15 2015 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/fs/FsShell.java | 11 -
.../java/org/apache/hadoop/fs/TestFsShell.java | 24
.../apache/hadoop/tracing/SetSpanReceiver.java | 2 +-
4 files changed, 38 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b09573/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index aeaa5b9..84e7076 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1020,6 +1020,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12245. References to misspelled REMAINING_QUATA in
FileSystemShell.md. (Gabor Liptak via aajisaka)
+HADOOP-12175. FsShell must load SpanReceierHost to support tracing
+(Masatake Iwasaki via Colin P. McCabe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b09573/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index 24b6339..e9c2f73 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.shell.Command;
import org.apache.hadoop.fs.shell.CommandFactory;
import org.apache.hadoop.fs.shell.FsCommand;
+import org.apache.hadoop.tracing.SpanReceiverHost;
import org.apache.hadoop.tools.TableListing;
import org.apache.hadoop.tracing.TraceUtils;
import org.apache.hadoop.util.Tool;
@@ -57,6 +58,9 @@ public class FsShell extends Configured implements Tool {
private final String usagePrefix =
"Usage: hadoop fs [generic options]";
+ private SpanReceiverHost spanReceiverHost;
+ static final String SEHLL_HTRACE_PREFIX = "dfs.shell.htrace.";
+
/**
* Default ctor with no configuration. Be sure to invoke
* {@link #setConf(Configuration)} with a valid configuration prior
@@ -97,6 +101,8 @@ public class FsShell extends Configured implements Tool {
commandFactory.addObject(new Usage(), "-usage");
registerCommands(commandFactory);
}
+this.spanReceiverHost =
+SpanReceiverHost.get(getConf(), SEHLL_HTRACE_PREFIX);
}
protected void registerCommands(CommandFactory factory) {
@@ -279,7 +285,7 @@ public class FsShell extends Configured implements Tool {
// initialize FsShell
init();
traceSampler = new SamplerBuilder(TraceUtils.
-wrapHadoopConf("dfs.shell.htrace.", getConf())).build();
+wrapHadoopConf(SEHLL_HTRACE_PREFIX, getConf())).build();
int exitCode = -1;
if (argv.length < 1) {
printUsage(System.err);
@@ -335,6 +341,9 @@ public class FsShell extends Configured implements Tool {
fs.close();
fs = null;
}
+if (this.spanReceiverHost != null) {
+ this.spanReceiverHost.closeReceivers();
+}
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b09573/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
index 12c8bcf..38e768f 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
@@ -18,6 +18,12 @@
package org.apache.hadoop.fs;
import junit.framework.AssertionFailedError;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop
[39/50] [abbrv] hadoop git commit: YARN-3853. Add docker container runtime support to LinuxContainterExecutor. Contributed by Sidharta Seethana.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
new file mode 100644
index 000..f9a890e
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -0,0 +1,107 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+
+import org.apache.hadoop.util.StringUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DockerRunCommand extends DockerCommand {
+ private static final String RUN_COMMAND = "run";
+ private final String image;
+ private List overrrideCommandWithArgs;
+
+ /** The following are mandatory: */
+ public DockerRunCommand(String containerId, String user, String image) {
+super(RUN_COMMAND);
+super.addCommandArguments("--name=" + containerId, "--user=" + user);
+this.image = image;
+ }
+
+ public DockerRunCommand removeContainerOnExit() {
+super.addCommandArguments("--rm");
+return this;
+ }
+
+ public DockerRunCommand detachOnRun() {
+super.addCommandArguments("-d");
+return this;
+ }
+
+ public DockerRunCommand setContainerWorkDir(String workdir) {
+super.addCommandArguments("--workdir=" + workdir);
+return this;
+ }
+
+ public DockerRunCommand setNetworkType(String type) {
+super.addCommandArguments("--net=" + type);
+return this;
+ }
+
+ public DockerRunCommand addMountLocation(String sourcePath, String
+ destinationPath) {
+super.addCommandArguments("-v", sourcePath + ":" + destinationPath);
+return this;
+ }
+
+ public DockerRunCommand setCGroupParent(String parentPath) {
+super.addCommandArguments("--cgroup-parent=" + parentPath);
+return this;
+ }
+
+ public DockerRunCommand addDevice(String sourceDevice, String
+ destinationDevice) {
+super.addCommandArguments("--device=" + sourceDevice + ":" +
+destinationDevice);
+return this;
+ }
+
+ public DockerRunCommand enableDetach() {
+super.addCommandArguments("--detach=true");
+return this;
+ }
+
+ public DockerRunCommand disableDetach() {
+super.addCommandArguments("--detach=false");
+return this;
+ }
+
+ public DockerRunCommand setOverrideCommandWithArgs(
+ List overrideCommandWithArgs) {
+this.overrrideCommandWithArgs = overrideCommandWithArgs;
+return this;
+ }
+
+ @Override
+ public String getCommandWithArguments() {
+List argList = new ArrayList<>();
+
+argList.add(super.getCommandWithArguments());
+argList.add(image);
+
+if (overrrideCommandWithArgs != null) {
+ argList.addAll(overrrideCommandWithArgs);
+}
+
+return StringUtils.join(" ", argList);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerExecutionException.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerExecutionException.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containe
[05/50] [abbrv] hadoop git commit: HDFS-8797. WebHdfsFileSystem creates too many connections for pread. Contributed by Jing Zhao.
HDFS-8797. WebHdfsFileSystem creates too many connections for pread.
Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e91ccfad
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e91ccfad
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e91ccfad
Branch: refs/heads/YARN-1197
Commit: e91ccfad07ec5b5674a84009772dd31a82b4e4de
Parents: 06e5dd2
Author: Jing Zhao
Authored: Wed Jul 22 17:42:31 2015 -0700
Committer: Jing Zhao
Committed: Wed Jul 22 17:42:31 2015 -0700
--
.../hadoop/hdfs/web/ByteRangeInputStream.java | 57 +---
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../hdfs/web/TestByteRangeInputStream.java | 35 ++--
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 41 ++
4 files changed, 113 insertions(+), 22 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e91ccfad/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java
index 395c9f6..bb581db 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/ByteRangeInputStream.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.web;
+import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
@@ -65,6 +66,16 @@ public abstract class ByteRangeInputStream extends
FSInputStream {
final boolean resolved) throws IOException;
}
+ static class InputStreamAndFileLength {
+final Long length;
+final InputStream in;
+
+InputStreamAndFileLength(Long length, InputStream in) {
+ this.length = length;
+ this.in = in;
+}
+ }
+
enum StreamStatus {
NORMAL, SEEK, CLOSED
}
@@ -101,7 +112,9 @@ public abstract class ByteRangeInputStream extends
FSInputStream {
if (in != null) {
in.close();
}
-in = openInputStream();
+InputStreamAndFileLength fin = openInputStream(startPos);
+in = fin.in;
+fileLength = fin.length;
status = StreamStatus.NORMAL;
break;
case CLOSED:
@@ -111,20 +124,22 @@ public abstract class ByteRangeInputStream extends
FSInputStream {
}
@VisibleForTesting
- protected InputStream openInputStream() throws IOException {
+ protected InputStreamAndFileLength openInputStream(long startOffset)
+ throws IOException {
// Use the original url if no resolved url exists, eg. if
// it's the first time a request is made.
final boolean resolved = resolvedURL.getURL() != null;
final URLOpener opener = resolved? resolvedURL: originalURL;
-final HttpURLConnection connection = opener.connect(startPos, resolved);
+final HttpURLConnection connection = opener.connect(startOffset, resolved);
resolvedURL.setURL(getResolvedUrl(connection));
InputStream in = connection.getInputStream();
+final Long length;
final Map> headers = connection.getHeaderFields();
if (isChunkedTransferEncoding(headers)) {
// file length is not known
- fileLength = null;
+ length = null;
} else {
// for non-chunked transfer-encoding, get content-length
final String cl = connection.getHeaderField(HttpHeaders.CONTENT_LENGTH);
@@ -133,14 +148,14 @@ public abstract class ByteRangeInputStream extends
FSInputStream {
+ headers);
}
final long streamlength = Long.parseLong(cl);
- fileLength = startPos + streamlength;
+ length = startOffset + streamlength;
// Java has a bug with >2GB request streams. It won't bounds check
// the reads so the transfer blocks until the server times out
in = new BoundedInputStream(in, streamlength);
}
-return in;
+return new InputStreamAndFileLength(length, in);
}
private static boolean isChunkedTransferEncoding(
@@ -204,6 +219,36 @@ public abstract class ByteRangeInputStream extends
FSInputStream {
}
}
+ @Override
+ public int read(long position, byte[] buffer, int offset, int length)
+ throws IOException {
+try (InputStream in = openInputStream(position).in) {
+ return in.read(buffer, offset, length);
+}
+ }
+
+ @Override
+ public void readFully(long position, byte[] buffer, int offset, int length)
+ throws IOException {
+final InputStreamAndFile
[34/50] [abbrv] hadoop git commit: YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. (Jonathan Yaniv and Ishai Menache via curino)
YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. (Jonathan Yaniv and Ishai Menache via curino) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/156f24ea Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/156f24ea Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/156f24ea Branch: refs/heads/YARN-1197 Commit: 156f24ead00436faad5d4aeef327a546392cd265 Parents: adcf5dd Author: ccurino Authored: Sat Jul 25 07:39:47 2015 -0700 Committer: ccurino Committed: Sat Jul 25 07:39:47 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 + .../reservation/AbstractReservationSystem.java | 2 + .../reservation/GreedyReservationAgent.java | 390 - .../reservation/InMemoryPlan.java | 13 +- .../InMemoryReservationAllocation.java | 8 +- .../resourcemanager/reservation/Plan.java | 1 + .../reservation/PlanContext.java| 2 + .../resourcemanager/reservation/PlanView.java | 31 +- .../resourcemanager/reservation/Planner.java| 47 -- .../RLESparseResourceAllocation.java| 55 +- .../reservation/ReservationAgent.java | 72 -- .../ReservationSchedulerConfiguration.java | 6 +- .../reservation/ReservationSystem.java | 5 +- .../reservation/ReservationSystemUtil.java | 6 +- .../reservation/SimpleCapacityReplanner.java| 113 --- .../planning/AlignedPlannerWithGreedy.java | 123 +++ .../planning/GreedyReservationAgent.java| 97 +++ .../reservation/planning/IterativePlanner.java | 338 .../reservation/planning/Planner.java | 49 ++ .../reservation/planning/PlanningAlgorithm.java | 207 + .../reservation/planning/ReservationAgent.java | 73 ++ .../planning/SimpleCapacityReplanner.java | 118 +++ .../reservation/planning/StageAllocator.java| 55 ++ .../planning/StageAllocatorGreedy.java | 152 .../planning/StageAllocatorLowCostAligned.java | 360 .../planning/StageEarliestStart.java| 46 ++ .../planning/StageEarliestStartByDemand.java| 106 +++ .../StageEarliestStartByJobArrival.java | 39 + .../planning/TryManyReservationAgents.java | 114 +++ .../reservation/ReservationSystemTestUtil.java | 5 +- .../reservation/TestCapacityOverTimePolicy.java | 2 +- .../TestCapacitySchedulerPlanFollower.java | 1 + .../reservation/TestFairReservationSystem.java | 1 - .../TestFairSchedulerPlanFollower.java | 1 + .../reservation/TestGreedyReservationAgent.java | 604 -- .../reservation/TestInMemoryPlan.java | 2 + .../reservation/TestNoOverCommitPolicy.java | 1 + .../TestRLESparseResourceAllocation.java| 51 +- .../TestSchedulerPlanFollowerBase.java | 1 + .../TestSimpleCapacityReplanner.java| 162 .../planning/TestAlignedPlanner.java| 820 +++ .../planning/TestGreedyReservationAgent.java| 611 ++ .../planning/TestSimpleCapacityReplanner.java | 170 43 files changed, 3634 insertions(+), 1429 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/156f24ea/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 55258a6..883d009 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -147,6 +147,9 @@ Release 2.8.0 - UNRELEASED YARN-2019. Retrospect on decision of making RM crashed if any exception throw in ZKRMStateStore. (Jian He via junping_du) +YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. +(Jonathan Yaniv and Ishai Menache via curino) + IMPROVEMENTS YARN-644. Basic null check is not performed on passed in arguments before http://git-wip-us.apache.org/repos/asf/hadoop/blob/156f24ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java index 8a15ac6..d2603c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/serv
[33/50] [abbrv] hadoop git commit: YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. (Jonathan Yaniv and Ishai Menache via curino)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/156f24ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
new file mode 100644
index 000..9a0a0f0
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/PlanningAlgorithm.java
@@ -0,0 +1,207 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryReservationAllocation;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.Plan;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.RLESparseResourceAllocation;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ContractValidationException;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+
+/**
+ * An abstract class that follows the general behavior of planning algorithms.
+ */
+public abstract class PlanningAlgorithm implements ReservationAgent {
+
+ /**
+ * Performs the actual allocation for a ReservationDefinition within a Plan.
+ *
+ * @param reservationId the identifier of the reservation
+ * @param user the user who owns the reservation
+ * @param plan the Plan to which the reservation must be fitted
+ * @param contract encapsulates the resources required by the user for his
+ * session
+ * @param oldReservation the existing reservation (null if none)
+ * @return whether the allocateUser function was successful or not
+ *
+ * @throws PlanningException if the session cannot be fitted into the plan
+ * @throws ContractValidationException
+ */
+ protected boolean allocateUser(ReservationId reservationId, String user,
+ Plan plan, ReservationDefinition contract,
+ ReservationAllocation oldReservation) throws PlanningException,
+ ContractValidationException {
+
+// Adjust the ResourceDefinition to account for system "imperfections"
+// (e.g., scheduling delays for large containers).
+ReservationDefinition adjustedContract = adjustContract(plan, contract);
+
+// Compute the job allocation
+RLESparseResourceAllocation allocation =
+computeJobAllocation(plan, reservationId, adjustedContract);
+
+// If no job allocation was found, fail
+if (allocation == null) {
+ throw new PlanningException(
+ "The planning algorithm could not find a valid allocation"
+ + " for your request");
+}
+
+// Translate the allocation to a map (with zero paddings)
+long step = plan.getStep();
+long jobArrival = stepRoundUp(adjustedContract.getArrival(), step);
+long jobDeadline = stepRoundUp(adjustedContract.getDeadline(), step);
+Map mapAllocations =
+allocationsToPaddedMap(allocation, jobArrival, jobDeadline);
+
+// Create the reservation
+ReservationAllocation capReservation =
+new InMemoryReservationAllocation(reservationId, // ID
+adjustedContract, // Contract
+user, // User name
+plan.getQueueName()
[49/50] [abbrv] hadoop git commit: YARN-1645. ContainerManager implementation to support container resizing. Contributed by Meng Ding & Wangda Tan
YARN-1645. ContainerManager implementation to support container resizing.
Contributed by Meng Ding & Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4facdcec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4facdcec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4facdcec
Branch: refs/heads/YARN-1197
Commit: 4facdcec910a86e043a065ee26ca73f6bbd836f0
Parents: b3f2830
Author: Jian He
Authored: Tue Jul 21 16:10:40 2015 -0700
Committer: Jian He
Committed: Tue Jul 28 13:53:32 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../CMgrDecreaseContainersResourceEvent.java| 37
.../nodemanager/ContainerManagerEventType.java | 1 +
.../containermanager/ContainerManagerImpl.java | 180 --
.../container/ChangeContainerResourceEvent.java | 36
.../container/ContainerEventType.java | 4 +
.../nodemanager/DummyContainerManager.java | 6 +-
.../TestContainerManagerWithLCE.java| 22 +++
.../BaseContainerManagerTest.java | 43 -
.../containermanager/TestContainerManager.java | 190 ++-
10 files changed, 486 insertions(+), 36 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4facdcec/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b09362e..eace759 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -162,6 +162,9 @@ Release 2.8.0 - UNRELEASED
YARN-1449. AM-NM protocol changes to support container resizing.
(Meng Ding & Wangda Tan via jianhe)
+YARN-1645. ContainerManager implementation to support container resizing.
+(Meng Ding & Wangda Tan via jianhe)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4facdcec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
new file mode 100644
index 000..9479d0b
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import java.util.List;
+
+public class CMgrDecreaseContainersResourceEvent extends ContainerManagerEvent
{
+
+ private final List containersToDecrease;
+
+ public CMgrDecreaseContainersResourceEvent(List
+ containersToDecrease) {
+super(ContainerManagerEventType.DECREASE_CONTAINERS_RESOURCE);
+this.containersToDecrease = containersToDecrease;
+ }
+
+ public List getContainersToDecrease() {
+return this.containersToDecrease;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4facdcec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
b/hadoop-yarn-project/hadoop-yarn/h
[31/50] [abbrv] hadoop git commit: YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. (Jonathan Yaniv and Ishai Menache via curino)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/156f24ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
new file mode 100644
index 000..bd18a2f
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/planning/TestGreedyReservationAgent.java
@@ -0,0 +1,611 @@
+/***
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+
***/
+package org.apache.hadoop.yarn.server.resourcemanager.reservation.planning;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+
+import org.apache.hadoop.yarn.api.records.ReservationDefinition;
+import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ReservationRequest;
+import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
+import org.apache.hadoop.yarn.api.records.ReservationRequests;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.CapacityOverTimePolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryPlan;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.InMemoryReservationAllocation;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationAllocation;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationInterval;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil;
+import
org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.Before;
+import org.junit.Test;
+import org.mortbay.log.Log;
+
+public class TestGreedyReservationAgent {
+
+ ReservationAgent agent;
+ InMemoryPlan plan;
+ Resource minAlloc = Resource.newInstance(1024, 1);
+ ResourceCalculator res = new DefaultResourceCalculator();
+ Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
+ Random rand = new Random();
+ long step;
+
+ @Before
+ public void setup() throws Exception {
+
+long seed = rand.nextLong();
+rand.setSeed(seed);
+Log.info("Running with seed: " + seed);
+
+// setting completely loose quotas
+long timeWindow = 100L;
+Resource clusterCapacity = Resource.newInstance(100 * 1024, 100);
+step = 1000L;
+ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
+String reservationQ = testUtil.getFullReservationQueueName();
+
+float instConstraint = 100;
+float a
[26/50] [abbrv] hadoop git commit: YARN-3973. Recent changes to application priority management break reservation system from YARN-1051 (Carlo Curino via wangda)
YARN-3973. Recent changes to application priority management break reservation
system from YARN-1051 (Carlo Curino via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a3bd7b4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a3bd7b4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a3bd7b4a
Branch: refs/heads/YARN-1197
Commit: a3bd7b4a59b3664273dc424f240356838213d4e7
Parents: ff9c13e
Author: Wangda Tan
Authored: Fri Jul 24 16:44:18 2015 -0700
Committer: Wangda Tan
Committed: Fri Jul 24 16:44:18 2015 -0700
--
hadoop-yarn-project/CHANGES.txt| 6 +-
.../resourcemanager/scheduler/capacity/CapacityScheduler.java | 2 +-
2 files changed, 6 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3bd7b4a/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c295784..55258a6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -667,7 +667,8 @@ Release 2.8.0 - UNRELEASED
YARN-3956. Fix TestNodeManagerHardwareUtils fails on Mac (Varun Vasudev
via wangda)
-YARN-3941. Proportional Preemption policy should try to avoid sending
duplicate PREEMPT_CONTAINER event to scheduler. (Sunil G via wangda)
+YARN-3941. Proportional Preemption policy should try to avoid sending
duplicate
+PREEMPT_CONTAINER event to scheduler. (Sunil G via wangda)
YARN-3900. Protobuf layout of yarn_security_token causes errors in other
protos
that include it (adhoot via rkanter)
@@ -678,6 +679,9 @@ Release 2.8.0 - UNRELEASED
YARN-3957. FairScheduler NPE In FairSchedulerQueueInfo causing scheduler
page to
return 500. (Anubhav Dhoot via kasha)
+YARN-3973. Recent changes to application priority management break
+reservation system from YARN-1051. (Carlo Curino via wangda)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a3bd7b4a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 68e608a..0b39d35 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1867,7 +1867,7 @@ public class CapacityScheduler extends
private Priority getDefaultPriorityForQueue(String queueName) {
Queue queue = getQueue(queueName);
-if (null == queue) {
+if (null == queue || null == queue.getDefaultApplicationPriority()) {
// Return with default application priority
return Priority.newInstance(CapacitySchedulerConfiguration
.DEFAULT_CONFIGURATION_APPLICATION_PRIORITY);
[16/50] [abbrv] hadoop git commit: HDFS-8806. Inconsistent metrics: number of missing blocks with replication factor 1 not properly cleared. Contributed by Zhe Zhang.
HDFS-8806. Inconsistent metrics: number of missing blocks with replication
factor 1 not properly cleared. Contributed by Zhe Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/206d4933
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/206d4933
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/206d4933
Branch: refs/heads/YARN-1197
Commit: 206d4933a567147b62f463c2daa3d063ad40822b
Parents: e202efa
Author: Akira Ajisaka
Authored: Fri Jul 24 18:28:44 2015 +0900
Committer: Akira Ajisaka
Committed: Fri Jul 24 18:28:44 2015 +0900
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/206d4933/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f86d41e..b348a5a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1097,6 +1097,9 @@ Release 2.7.2 - UNRELEASED
HDFS-6945. BlockManager should remove a block from excessReplicateMap and
decrement ExcessBlocks metric when the block is removed. (aajisaka)
+HDFS-8806. Inconsistent metrics: number of missing blocks with replication
+factor 1 not properly cleared. (Zhe Zhang via aajisaka)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/206d4933/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
index d8aec99..128aae6 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
@@ -101,10 +101,11 @@ class UnderReplicatedBlocks implements
Iterable {
/**
* Empty the queues and timestamps.
*/
- void clear() {
+ synchronized void clear() {
for (int i = 0; i < LEVEL; i++) {
priorityQueues.get(i).clear();
}
+corruptReplOneBlocks = 0;
timestampsMap.clear();
}
[18/50] [abbrv] hadoop git commit: YARN-3967. Fetch the application report from the AHS if the RM does not know about it. Contributed by Mit Desai
YARN-3967. Fetch the application report from the AHS if the RM does not
know about it. Contributed by Mit Desai
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fbd60632
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fbd60632
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fbd60632
Branch: refs/heads/YARN-1197
Commit: fbd6063269221ec25834684477f434e19f0b66af
Parents: ee233ec
Author: Xuan
Authored: Fri Jul 24 10:15:54 2015 -0700
Committer: Xuan
Committed: Fri Jul 24 10:15:54 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/server/webproxy/AppReportFetcher.java | 79 +++--
.../server/webproxy/TestAppReportFetcher.java | 117 +++
3 files changed, 187 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbd60632/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8bc9e4c..a25387d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -683,6 +683,9 @@ Release 2.7.2 - UNRELEASED
YARN-3170. YARN architecture document needs updating. (Brahma Reddy Battula
via ozawa)
+YARN-3967. Fetch the application report from the AHS if the RM does not
know about it.
+(Mit Desai via xgong)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fbd60632/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
index 5c93413..6aa43eb 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/AppReportFetcher.java
@@ -24,11 +24,15 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.ApplicationHistoryProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.client.AHSProxy;
import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -41,38 +45,73 @@ public class AppReportFetcher {
private static final Log LOG = LogFactory.getLog(AppReportFetcher.class);
private final Configuration conf;
private final ApplicationClientProtocol applicationsManager;
+ private final ApplicationHistoryProtocol historyManager;
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
-
+ private boolean isAHSEnabled;
+
/**
- * Create a new Connection to the RM to fetch Application reports.
+ * Create a new Connection to the RM/Application History Server
+ * to fetch Application reports.
* @param conf the conf to use to know where the RM is.
*/
public AppReportFetcher(Configuration conf) {
+if (conf.getBoolean(YarnConfiguration.APPLICATION_HISTORY_ENABLED,
+YarnConfiguration.DEFAULT_APPLICATION_HISTORY_ENABLED)) {
+ isAHSEnabled = true;
+}
this.conf = conf;
try {
applicationsManager = ClientRMProxy.createRMProxy(conf,
ApplicationClientProtocol.class);
+ if (isAHSEnabled) {
+historyManager = getAHSProxy(conf);
+ } else {
+this.historyManager = null;
+ }
} catch (IOException e) {
throw new YarnRuntimeException(e);
}
}
/**
- * Just call directly into the applicationsManager given instead of creating
- * a remote connection to it. This is mostly for when the Proxy is running
- * as part of
[29/50] [abbrv] hadoop git commit: HADOOP-12202. releasedocmaker drops missing component and assignee entries (aw)
HADOOP-12202. releasedocmaker drops missing component and assignee entries (aw)
(cherry picked from commit adbacf7010373dbe6df239688b4cebd4a93a69e4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d7697831
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d7697831
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d7697831
Branch: refs/heads/YARN-1197
Commit: d7697831e3b24bec149990feed819e7d96f78184
Parents: e8b62d1
Author: Allen Wittenauer
Authored: Tue Jul 7 14:30:32 2015 -0700
Committer: Andrew Wang
Committed: Fri Jul 24 18:31:44 2015 -0700
--
dev-support/releasedocmaker.py | 24
1 file changed, 12 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d7697831/dev-support/releasedocmaker.py
--
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index 6e01260..409d8e3 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -420,6 +420,8 @@ def main():
else:
title=options.title
+ haderrors=False
+
for v in versions:
vstr=str(v)
jlist = JiraIter(vstr,projects)
@@ -468,14 +470,6 @@ def main():
for jira in sorted(jlist):
if jira.getIncompatibleChange():
incompatlist.append(jira)
-if (len(jira.getReleaseNote())==0):
-warningCount+=1
-
- if jira.checkVersionString():
- warningCount+=1
-
- if jira.checkMissingComponent() or jira.checkMissingAssignee():
-errorCount+=1
elif jira.getType() == "Bug":
buglist.append(jira)
elif jira.getType() == "Improvement":
@@ -496,6 +490,7 @@ def main():
notableclean(jira.getSummary()))
if (jira.getIncompatibleChange()) and (len(jira.getReleaseNote())==0):
+warningCount+=1
reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
reloutputs.writeKeyRaw(jira.getProject(), line)
line ='\n**WARNING: No release note provided for this incompatible
change.**\n\n'
@@ -503,9 +498,11 @@ def main():
reloutputs.writeKeyRaw(jira.getProject(), line)
if jira.checkVersionString():
+ warningCount+=1
lintMessage += "\nWARNING: Version string problem for %s " %
jira.getId()
if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
+ errorCount+=1
errorMessage=[]
jira.checkMissingComponent() and errorMessage.append("component")
jira.checkMissingAssignee() and errorMessage.append("assignee")
@@ -520,11 +517,11 @@ def main():
if (options.lint is True):
print lintMessage
print "==="
-print "Error:%d, Warning:%d \n" % (errorCount, warningCount)
-
+print "%s: Error:%d, Warning:%d \n" % (vstr, errorCount, warningCount)
if (errorCount>0):
-cleanOutputDir(version)
-sys.exit(1)
+ haderrors=True
+ cleanOutputDir(vstr)
+ continue
reloutputs.writeAll("\n\n")
reloutputs.close()
@@ -571,5 +568,8 @@ def main():
if options.index:
buildindex(title,options.license)
+ if haderrors is True:
+sys.exit(1)
+
if __name__ == "__main__":
main()
[44/50] [abbrv] hadoop git commit: YARN-3982. container-executor parsing of container-executor.cfg broken in trunk and branch-2. Contributed by Varun Vasudev
YARN-3982. container-executor parsing of container-executor.cfg broken
in trunk and branch-2. Contributed by Varun Vasudev
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f1709342
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f1709342
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f1709342
Branch: refs/heads/YARN-1197
Commit: f17093421521efcbdc813f6f2b8411e45ecc7863
Parents: 030fcfa
Author: Xuan
Authored: Mon Jul 27 23:45:58 2015 -0700
Committer: Xuan
Committed: Mon Jul 27 23:45:58 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../container-executor/impl/configuration.c | 4 ++--
.../test/test-container-executor.c | 22 +---
3 files changed, 20 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4f8484a..b4666e8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -698,6 +698,9 @@ Release 2.8.0 - UNRELEASED
YARN-3846. RM Web UI queue filter is not working for sub queue.
(Mohammad Shahid Khan via jianhe)
+YARN-3982. container-executor parsing of container-executor.cfg broken in
+trunk and branch-2. (Varun Vasudev via xgong)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index 2825367..373dbfd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -284,11 +284,11 @@ char * get_value(const char* key) {
/**
* Function to return an array of values for a key.
- * Value delimiter is assumed to be a '%'.
+ * Value delimiter is assumed to be a ','.
*/
char ** get_values(const char * key) {
char *value = get_value(key);
- return extract_values(value);
+ return extract_values_delim(value, ",");
}
char ** extract_values_delim(char *value, const char *delim) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 99bcf34..001a37d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -145,7 +145,7 @@ void check_pid_file(const char* pid_file, pid_t mypid) {
}
char myPidBuf[33];
- snprintf(myPidBuf, 33, "%" PRId64, (int64_t)mypid);
+ snprintf(myPidBuf, 33, "%" PRId64, (int64_t)(mypid + 1));
if (strncmp(pidBuf, myPidBuf, strlen(myPidBuf)) != 0) {
printf("FAIL: failed to find matching pid in pid file\n");
printf("FAIL: Expected pid %" PRId64 " : Got %.*s", (int64_t)mypid,
@@ -212,15 +212,15 @@ void test_get_app_log_dir() {
free(logdir);
}
-void test_check_user() {
+void test_check_user(int expectedFailure) {
printf("\nTesting test_check_user\n");
struct passwd *user = check_user(username);
- if (user == NULL) {
+ if (user == NULL && !expectedFailure) {
printf("FAIL: failed check for user %s\n", username);
exit(1);
}
free(user);
- if (check_user("lp") != NULL) {
+ if (check_user("lp") != NULL && !expectedFailure) {
printf("FAIL: failed check for system user lp\n");
exit(1);
}
@@ -228,7 +228,7 @@ void test_check_user() {
printf("FAIL: failed check for system user root\n
[08/50] [abbrv] hadoop git commit: HADOOP-12161. Add getStoragePolicy API to the FileSystem interface. (Contributed by Brahma Reddy Battula)
HADOOP-12161. Add getStoragePolicy API to the FileSystem interface.
(Contributed by Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adfa34ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adfa34ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adfa34ff
Branch: refs/heads/YARN-1197
Commit: adfa34ff9992295a6d2496b259d8c483ed90b566
Parents: 3bba180
Author: Arpit Agarwal
Authored: Thu Jul 23 10:13:04 2015 -0700
Committer: Arpit Agarwal
Committed: Thu Jul 23 10:13:04 2015 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../apache/hadoop/fs/AbstractFileSystem.java| 13 +
.../java/org/apache/hadoop/fs/FileContext.java | 20
.../java/org/apache/hadoop/fs/FileSystem.java | 13 +
.../org/apache/hadoop/fs/FilterFileSystem.java | 6 ++
.../java/org/apache/hadoop/fs/FilterFs.java | 6 ++
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 6 ++
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 15 +++
.../org/apache/hadoop/fs/TestHarFileSystem.java | 3 +++
.../main/java/org/apache/hadoop/fs/Hdfs.java| 5 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 18 ++
.../hadoop/hdfs/DistributedFileSystem.java | 19 +++
.../hadoop/hdfs/TestBlockStoragePolicy.java | 17 +
13 files changed, 144 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index ff7d2ad..f1a3bc9 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -716,6 +716,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12184. Remove unused Linux-specific constants in NativeIO (Martin
Walsh via Colin P. McCabe)
+HADOOP-12161. Add getStoragePolicy API to the FileSystem interface.
+(Brahma Reddy Battula via Arpit Agarwal)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
index cb3fb86..2bc3859 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/AbstractFileSystem.java
@@ -1237,6 +1237,19 @@ public abstract class AbstractFileSystem {
}
/**
+ * Retrieve the storage policy for a given file or directory.
+ *
+ * @param src file or directory path.
+ * @return storage policy for give file.
+ * @throws IOException
+ */
+ public BlockStoragePolicySpi getStoragePolicy(final Path src)
+ throws IOException {
+throw new UnsupportedOperationException(getClass().getSimpleName()
++ " doesn't support getStoragePolicy");
+ }
+
+ /**
* Retrieve all the storage policies supported by this file system.
*
* @return all storage policies supported by this filesystem.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adfa34ff/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index 0f21a61..a98d662 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static
org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
+
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RpcClientException;
import org.apache.hadoop.ipc.RpcServerException;
@@ -2692,6 +2693,25 @@ public class FileContext {
}
/**
+ * Query the effective storage policy ID for the given file or directory.
+ *
+ * @param src file or direc
[43/50] [abbrv] hadoop git commit: HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)
HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/030fcfa9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/030fcfa9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/030fcfa9
Branch: refs/heads/YARN-1197
Commit: 030fcfa99c345ad57625486eeabedebf2fd4411f
Parents: e21dde5
Author: Arun Suresh
Authored: Mon Jul 27 23:02:03 2015 -0700
Committer: Arun Suresh
Committed: Mon Jul 27 23:02:03 2015 -0700
--
.../apache/hadoop/io/retry/MultiException.java | 49 +++
.../hadoop/io/retry/RetryInvocationHandler.java | 99 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../ha/ConfiguredFailoverProxyProvider.java | 52 ++-
.../ha/RequestHedgingProxyProvider.java | 186 ++
.../markdown/HDFSHighAvailabilityWithNFS.md | 9 +-
.../markdown/HDFSHighAvailabilityWithQJM.md | 10 +-
.../ha/TestRequestHedgingProxyProvider.java | 350 +++
8 files changed, 724 insertions(+), 33 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/030fcfa9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
new file mode 100644
index 000..4963a2d
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.io.retry;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Holder class that clients can use to return multiple exceptions.
+ */
+public class MultiException extends IOException {
+
+ private final Map exes;
+
+ public MultiException(Map exes) {
+this.exes = exes;
+ }
+
+ public Map getExceptions() {
+return exes;
+ }
+
+ @Override
+ public String toString() {
+StringBuilder sb = new StringBuilder("{");
+for (Exception e : exes.values()) {
+ sb.append(e.toString()).append(", ");
+}
+sb.append("}");
+return "MultiException[" + sb.toString() + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/030fcfa9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
index 543567e..9256356 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
@@ -23,6 +23,8 @@ import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -101,7 +103,7 @@ public class RetryInvocationHandler implements
RpcInvocationHandler {
Object ret = invokeMethod(method, args);
hasMadeASuccessfulCall = true;
return ret;
- } catch (Exception e) {
+ } catch (Exception ex) {
boolean isIdempotentOrAtMostOnce = proxyProvider.getInterface()
.getMethod(method.getName(), method.getParameterTypes())
.isAnnotationPresent(Idempotent.class);
@@ -110,15 +112,16 @@ public class RetryInvocationHandler implements
RpcInvocationHandler {
.getMethod(method.getName(), method.getParameterTypes())
.isA
[12/50] [abbrv] hadoop git commit: HADOOP-12009: Clarify FileSystem.listStatus() sorting order & fix FileSystemContractBaseTest:testListStatus. (J.Andreina via jghoman)
HADOOP-12009: Clarify FileSystem.listStatus() sorting order & fix
FileSystemContractBaseTest:testListStatus. (J.Andreina via jghoman)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab3197c2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab3197c2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab3197c2
Branch: refs/heads/YARN-1197
Commit: ab3197c20452e0dd908193d6854c204e6ee34645
Parents: 1d3026e
Author: Jakob Homan
Authored: Thu Jul 23 17:46:13 2015 -0700
Committer: Jakob Homan
Committed: Thu Jul 23 17:46:13 2015 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
.../main/java/org/apache/hadoop/fs/FileSystem.java | 17 -
.../src/site/markdown/filesystem/filesystem.md | 4
.../hadoop/fs/FileSystemContractBaseTest.java | 11 ---
4 files changed, 31 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab3197c2/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 6c18add..56edcac 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -497,6 +497,9 @@ Trunk (Unreleased)
HADOOP-11762. Enable swift distcp to secure HDFS (Chen He via aw)
+HADOOP-12009. Clarify FileSystem.listStatus() sorting order & fix
+FileSystemContractBaseTest:testListStatus. (J.Andreina via jghoman)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab3197c2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index a01d3ea..8f32644 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1501,7 +1501,9 @@ public abstract class FileSystem extends Configured
implements Closeable {
/**
* List the statuses of the files/directories in the given path if the path
is
* a directory.
- *
+ *
+ * Does not guarantee to return the List of files/directories status in a
+ * sorted order.
* @param f given path
* @return the statuses of the files/directories in the given patch
* @throws FileNotFoundException when the path does not exist;
@@ -1543,6 +1545,9 @@ public abstract class FileSystem extends Configured
implements Closeable {
/**
* Filter files/directories in the given path using the user-supplied path
* filter.
+ *
+ * Does not guarantee to return the List of files/directories status in a
+ * sorted order.
*
* @param f
* a path name
@@ -1563,6 +1568,9 @@ public abstract class FileSystem extends Configured
implements Closeable {
/**
* Filter files/directories in the given list of paths using default
* path filter.
+ *
+ * Does not guarantee to return the List of files/directories status in a
+ * sorted order.
*
* @param files
* a list of paths
@@ -1579,6 +1587,9 @@ public abstract class FileSystem extends Configured
implements Closeable {
/**
* Filter files/directories in the given list of paths using user-supplied
* path filter.
+ *
+ * Does not guarantee to return the List of files/directories status in a
+ * sorted order.
*
* @param files
* a list of paths
@@ -1739,6 +1750,8 @@ public abstract class FileSystem extends Configured
implements Closeable {
* while consuming the entries. Each file system implementation should
* override this method and provide a more efficient implementation, if
* possible.
+ * Does not guarantee to return the iterator that traverses statuses
+ * of the files in a sorted order.
*
* @param p target path
* @return remote iterator
@@ -1766,6 +1779,8 @@ public abstract class FileSystem extends Configured
implements Closeable {
/**
* List the statuses and block locations of the files in the given path.
+ * Does not guarantee to return the iterator that traverses statuses
+ * of the files in a sorted order.
*
* If the path is a directory,
* if recursive is false, returns files in the directory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab3197c2/hadoop-common-project/hadoop-common/src/site/markdown/filesy
[20/50] [abbrv] hadoop git commit: YARN-3957. FairScheduler NPE In FairSchedulerQueueInfo causing scheduler page to return 500. (Anubhav Dhoot via kasha)
YARN-3957. FairScheduler NPE In FairSchedulerQueueInfo causing scheduler page
to return 500. (Anubhav Dhoot via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d19d1877
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d19d1877
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d19d1877
Branch: refs/heads/YARN-1197
Commit: d19d18775368f5aaa254881165acc1299837072b
Parents: f8f6091
Author: Karthik Kambatla
Authored: Fri Jul 24 11:44:37 2015 -0700
Committer: Karthik Kambatla
Committed: Fri Jul 24 11:44:37 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../webapp/dao/FairSchedulerQueueInfo.java | 4 +-
.../webapp/dao/TestFairSchedulerQueueInfo.java | 59
3 files changed, 65 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19d1877/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a25387d..44e5510 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -672,6 +672,9 @@ Release 2.8.0 - UNRELEASED
YARN-3845. Scheduler page does not render RGBA color combinations in IE11.
(Contributed by Mohammad Shahid Khan)
+YARN-3957. FairScheduler NPE In FairSchedulerQueueInfo causing scheduler
page to
+return 500. (Anubhav Dhoot via kasha)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19d1877/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index 9b297a2..7ba0988 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+import java.util.ArrayList;
import java.util.Collection;
import javax.xml.bind.annotation.XmlAccessType;
@@ -204,6 +205,7 @@ public class FairSchedulerQueueInfo {
}
public Collection getChildQueues() {
-return childQueues.getQueueInfoList();
+return childQueues != null ? childQueues.getQueueInfoList() :
+new ArrayList();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d19d1877/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
new file mode 100644
index 000..973afcf
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/TestFairSchedulerQueueInfo.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for
[41/50] [abbrv] hadoop git commit: YARN-3846. RM Web UI queue filter is not working for sub queue. Contributed by Mohammad Shahid Khan
YARN-3846. RM Web UI queue filter is not working for sub queue. Contributed by
Mohammad Shahid Khan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3572ebd7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3572ebd7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3572ebd7
Branch: refs/heads/YARN-1197
Commit: 3572ebd738aa5fa8b0906d75fb12cc6cbb991573
Parents: 3e6fce9
Author: Jian He
Authored: Mon Jul 27 16:57:11 2015 -0700
Committer: Jian He
Committed: Mon Jul 27 17:12:05 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/resourcemanager/webapp/CapacitySchedulerPage.java| 5 -
2 files changed, 7 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3572ebd7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 534c55a..4f8484a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -695,6 +695,9 @@ Release 2.8.0 - UNRELEASED
YARN-3958. TestYarnConfigurationFields should be moved to hadoop-yarn-api
module. (Varun Saxena via aajisaka)
+YARN-3846. RM Web UI queue filter is not working for sub queue.
+(Mohammad Shahid Khan via jianhe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3572ebd7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 12a3013..d8971b7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -516,7 +516,10 @@ class CapacitySchedulerPage extends RmView {
" $('#cs').bind('select_node.jstree', function(e, data) {",
"var q = $('.q', data.rslt.obj).first().text();",
"if (q == 'Queue: root') q = '';",
- "else q = '^' + q.substr(q.lastIndexOf(':') + 2) + '$';",
+ "else {",
+ " q = q.substr(q.lastIndexOf(':') + 2);",
+ " q = '^' + q.substr(q.lastIndexOf('.') + 1) + '$';",
+ "}",
"$('#apps').dataTable().fnFilter(q, 4, true);",
" });",
" $('#cs').show();",
[48/50] [abbrv] hadoop git commit: YARN-3866. AM-RM protocol changes to support container resizing. Contributed by Meng Ding
YARN-3866. AM-RM protocol changes to support container resizing. Contributed by
Meng Ding
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7758afd3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7758afd3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7758afd3
Branch: refs/heads/YARN-1197
Commit: 7758afd38469a7f265d5b48265b1290f75ee6b33
Parents: 69b0957
Author: Jian He
Authored: Mon Jul 13 17:34:26 2015 -0700
Committer: Jian He
Committed: Tue Jul 28 13:53:32 2015 -0700
--
.../app/local/TestLocalContainerAllocator.java | 6 +-
hadoop-yarn-project/CHANGES.txt | 2 +
.../api/protocolrecords/AllocateRequest.java| 57 --
.../api/protocolrecords/AllocateResponse.java | 42 +++--
.../records/ContainerResourceChangeRequest.java | 117 +
.../api/records/ContainerResourceDecrease.java | 78 -
.../api/records/ContainerResourceIncrease.java | 84 -
.../ContainerResourceIncreaseRequest.java | 80 -
.../yarn/api/records/ContainerStatus.java | 13 ++
.../src/main/proto/yarn_protos.proto| 14 +-
.../src/main/proto/yarn_service_protos.proto| 16 +-
.../impl/pb/AllocateRequestPBImpl.java | 119 ++---
.../impl/pb/AllocateResponsePBImpl.java | 175 +--
.../ContainerResourceChangeRequestPBImpl.java | 141 +++
.../pb/ContainerResourceDecreasePBImpl.java | 136 --
.../pb/ContainerResourceIncreasePBImpl.java | 171 --
.../ContainerResourceIncreaseRequestPBImpl.java | 141 ---
.../records/impl/pb/ContainerStatusPBImpl.java | 31 +++-
.../hadoop/yarn/api/TestAllocateRequest.java| 73
.../hadoop/yarn/api/TestAllocateResponse.java | 114
.../yarn/api/TestContainerResourceDecrease.java | 66 ---
.../yarn/api/TestContainerResourceIncrease.java | 74
.../TestContainerResourceIncreaseRequest.java | 68 ---
.../hadoop/yarn/api/TestPBImplRecords.java | 34 +---
24 files changed, 533 insertions(+), 1319 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7758afd3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
index f901ed8..167d804 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/local/TestLocalContainerAllocator.java
@@ -46,8 +46,6 @@ import
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRespo
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.ContainerResourceDecrease;
-import org.apache.hadoop.yarn.api.records.ContainerResourceIncrease;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -254,8 +252,8 @@ public class TestLocalContainerAllocator {
Resources.none(), null, 1, null,
Collections.emptyList(),
yarnToken,
- Collections.emptyList(),
- Collections.emptyList());
+ Collections.emptyList(),
+ Collections.emptyList());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7758afd3/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b4666e8..87f0e91 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -156,6 +156,8 @@ Release 2.8.0 - UNRELEASED
YARN-3853. Add docker container runtime support to LinuxContainterExecutor.
(Sidharta Seethana via vvasudev)
+YARN-3866. AM-RM protocol changes to support container resizing. (Meng Ding
+via jianhe)
IMPROVEMENTS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7758afd3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/mai
[50/50] [abbrv] hadoop git commit: YARN-3867. ContainerImpl changes to support container resizing. Contributed by Meng Ding
YARN-3867. ContainerImpl changes to support container resizing. Contributed by
Meng Ding
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb956626
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb956626
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb956626
Branch: refs/heads/YARN-1197
Commit: cb956626955a60a9d972b4376bc1b334d3b14993
Parents: 4facdce
Author: Jian He
Authored: Tue Jul 28 13:51:23 2015 -0700
Committer: Jian He
Committed: Tue Jul 28 13:53:33 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../hadoop/yarn/server/utils/BuilderUtils.java | 4 ++-
.../containermanager/ContainerManagerImpl.java | 7 ++--
.../container/ChangeContainerResourceEvent.java | 36 ---
.../containermanager/container/Container.java | 2 ++
.../container/ContainerEventType.java | 4 ---
.../container/ContainerImpl.java| 16 ++---
.../ChangeMonitoringContainerResourceEvent.java | 37
.../monitor/ContainersMonitorEventType.java | 3 +-
.../nodemanager/metrics/NodeManagerMetrics.java | 11 ++
.../nodemanager/TestNodeStatusUpdater.java | 2 +-
.../metrics/TestNodeManagerMetrics.java | 18 +++---
.../nodemanager/webapp/MockContainer.java | 4 +++
.../yarn/server/resourcemanager/MockNM.java | 2 +-
.../server/resourcemanager/NodeManager.java | 2 +-
.../resourcemanager/TestApplicationCleanup.java | 6 ++--
.../attempt/TestRMAppAttemptTransitions.java| 21 +++
.../capacity/TestCapacityScheduler.java | 2 +-
.../scheduler/fifo/TestFifoScheduler.java | 4 +--
.../security/TestAMRMTokens.java| 3 +-
20 files changed, 118 insertions(+), 69 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb956626/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index eace759..40f0be3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -165,6 +165,9 @@ Release 2.8.0 - UNRELEASED
YARN-1645. ContainerManager implementation to support container resizing.
(Meng Ding & Wangda Tan via jianhe)
+YARN-3867. ContainerImpl changes to support container resizing. (Meng Ding
+via jianhe)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb956626/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index f2146c8..5f92c0c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -200,13 +200,15 @@ public class BuilderUtils {
}
public static ContainerStatus newContainerStatus(ContainerId containerId,
- ContainerState containerState, String diagnostics, int exitStatus) {
+ ContainerState containerState, String diagnostics, int exitStatus,
+ Resource capability) {
ContainerStatus containerStatus = recordFactory
.newRecordInstance(ContainerStatus.class);
containerStatus.setState(containerState);
containerStatus.setContainerId(containerId);
containerStatus.setDiagnostics(diagnostics);
containerStatus.setExitStatus(exitStatus);
+containerStatus.setCapability(capability);
return containerStatus;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb956626/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.ja
[02/50] [abbrv] hadoop git commit: YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo should based on total-used-resources. (Bibin A Chundatt via wangda)
YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo
should based on total-used-resources. (Bibin A Chundatt via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76ec26de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76ec26de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76ec26de
Branch: refs/heads/YARN-1197
Commit: 76ec26de8099dc48ce3812c595b7ab857a600442
Parents: 1b3bceb
Author: Wangda Tan
Authored: Wed Jul 22 11:54:02 2015 -0700
Committer: Wangda Tan
Committed: Wed Jul 22 11:54:02 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/SchedulerApplicationAttempt.java | 2 +-
.../scheduler/capacity/LeafQueue.java | 8 ++-
.../TestCapacitySchedulerNodeLabelUpdate.java | 64
4 files changed, 74 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5100cdf..f751862 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -653,6 +653,9 @@ Release 2.8.0 - UNRELEASED
YARN-3885. ProportionalCapacityPreemptionPolicy doesn't preempt if queue
is
more than 2 level. (Ajith S via wangda)
+YARN-3932. SchedulerApplicationAttempt#getResourceUsageReport and UserInfo
+should based on total-used-resources. (Bibin A Chundatt via wangda)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index cf543bd..317e61c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -598,7 +598,7 @@ public class SchedulerApplicationAttempt implements
SchedulableEntity {
AggregateAppResourceUsage runningResourceUsage =
getRunningAggregateAppResourceUsage();
Resource usedResourceClone =
-Resources.clone(attemptResourceUsage.getUsed());
+Resources.clone(attemptResourceUsage.getAllUsed());
Resource reservedResourceClone =
Resources.clone(attemptResourceUsage.getReserved());
return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76ec26de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 0ce4d68..5c283f4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -439,7 +439,7 @@ public class LeafQueue extends AbstractCSQueue {
for (Map.Entry entry : users.entrySet()) {
User user = entry.getValue();
usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(user
- .getUsed()), user.getActiveApplications(), user
+ .getAllUsed()), user.getActiveApplications(), user
.getPendingApplications(), Resources.clone(user
.getConsumedA
[35/50] [abbrv] hadoop git commit: YARN-3958. TestYarnConfigurationFields should be moved to hadoop-yarn-api module. Contributed by Varun Saxena.
YARN-3958. TestYarnConfigurationFields should be moved to hadoop-yarn-api
module. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42d4e0ae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42d4e0ae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42d4e0ae
Branch: refs/heads/YARN-1197
Commit: 42d4e0ae99d162fde52902cb86e29f2c82a084c8
Parents: 156f24e
Author: Akira Ajisaka
Authored: Mon Jul 27 11:43:25 2015 +0900
Committer: Akira Ajisaka
Committed: Mon Jul 27 11:43:25 2015 +0900
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 34 +
.../yarn/conf/TestYarnConfigurationFields.java | 136 +++
.../yarn/conf/TestYarnConfigurationFields.java | 136 ---
4 files changed, 173 insertions(+), 136 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42d4e0ae/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 883d009..3b7d8a8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -685,6 +685,9 @@ Release 2.8.0 - UNRELEASED
YARN-3973. Recent changes to application priority management break
reservation system from YARN-1051. (Carlo Curino via wangda)
+YARN-3958. TestYarnConfigurationFields should be moved to hadoop-yarn-api
+module. (Varun Saxena via aajisaka)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42d4e0ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
index dc9c469..5c4156b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
@@ -62,9 +62,31 @@
com.google.protobuf
protobuf-java
+
+
+ org.apache.hadoop
+ hadoop-common
+ test-jar
+ test
+
+
+
+ junit
+ junit
+ test
+
+
+
+
${basedir}/../hadoop-yarn-common/src/main/resources
+
+ yarn-default.xml
+
+false
+
+
org.apache.hadoop
@@ -105,6 +127,18 @@
+
+
+maven-jar-plugin
+
+
+
+ test-jar
+
+test-compile
+
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42d4e0ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
new file mode 100644
index 000..e89a90d
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.conf;
+
+import java.util.HashSet;
+
+import org.apache.hadoop.conf.TestConfigurationFieldsBase;
+
+/**
+ * Unit test class to compare
+ * {@link org.apache.hadoop.yarn.conf.YarnConfiguration} and
+ * yarn-default.xml for missing properties. Currently only throws an error
+ * if the class is missing a property.
+ *
+ * Refer to {@link org.apache.hadoop.conf.TestConfigurationFieldsBase}
+ * for how this class works.
+ */
+public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
+
+ @Suppr
[28/50] [abbrv] hadoop git commit: HADOOP-12135. cleanup releasedocmaker
HADOOP-12135. cleanup releasedocmaker
(cherry picked from commit 3fee9f8d18dd60d83da674b3cfbefe666915fad8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8b62d11
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8b62d11
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8b62d11
Branch: refs/heads/YARN-1197
Commit: e8b62d11d460e9706e48df92a0b0a72f4a02d3f5
Parents: 098ba45
Author: Allen Wittenauer
Authored: Mon Jul 6 15:49:03 2015 -0700
Committer: Andrew Wang
Committed: Fri Jul 24 18:31:30 2015 -0700
--
dev-support/releasedocmaker.py | 384 +++-
1 file changed, 207 insertions(+), 177 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b62d11/dev-support/releasedocmaker.py
--
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index 8e68b3c..6e01260 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -19,6 +19,7 @@
from glob import glob
from optparse import OptionParser
from time import gmtime, strftime
+import pprint
import os
import re
import sys
@@ -99,23 +100,44 @@ def mstr(obj):
return ""
return unicode(obj)
-def buildindex(master):
+def buildindex(title,license):
versions=reversed(sorted(glob("[0-9]*.[0-9]*.[0-9]*")))
with open("index.md","w") as indexfile:
+if license is True:
+ indexfile.write(asflicense)
for v in versions:
- indexfile.write("* Apache Hadoop v%s\n" % (v))
+ indexfile.write("* %s v%s\n" % (title,v))
for k in ("Changes","Release Notes"):
-indexfile.write("* %s\n" %(k))
-indexfile.write("* [Combined %s](%s/%s.%s.html)\n" \
+indexfile.write("* %s (%s/%s.%s.html)\n" \
% (k,v,k.upper().replace(" ",""),v))
-if not master:
- indexfile.write("* [Hadoop Common
%s](%s/%s.HADOOP.%s.html)\n" \
-% (k,v,k.upper().replace(" ",""),v))
- for p in ("HDFS","MapReduce","YARN"):
-indexfile.write("* [%s %s](%s/%s.%s.%s.html)\n" \
- % (p,k,v,k.upper().replace(" ",""),p.upper(),v))
indexfile.close()
+class GetVersions:
+ """ yo """
+ def __init__(self,versions, projects):
+versions = versions
+projects = projects
+self.newversions = []
+pp = pprint.PrettyPrinter(indent=4)
+at=0
+end=1
+count=100
+versions.sort()
+print "Looking for %s through %s"%(versions[0],versions[-1])
+for p in projects:
+ resp =
urllib.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
+ data = json.loads(resp.read())
+ for d in data:
+if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <=
versions[-1]:
+ print "Adding %s to the list" % d['name']
+ self.newversions.append(d['name'])
+newlist=list(set(self.newversions))
+self.newversions=newlist
+
+ def getlist(self):
+ pp = pprint.PrettyPrinter(indent=4)
+ return(self.newversions)
+
class Version:
"""Represents a version number"""
def __init__(self, data):
@@ -261,8 +283,10 @@ class Jira:
class JiraIter:
"""An Iterator of JIRAs"""
- def __init__(self, versions):
-self.versions = versions
+ def __init__(self, version, projects):
+self.version = version
+self.projects = projects
+v=str(version).replace("-SNAPSHOT","")
resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field";)
data = json.loads(resp.read())
@@ -276,7 +300,7 @@ class JiraIter:
end=1
count=100
while (at < end):
- params = urllib.urlencode({'jql': "project in
(HADOOP,HDFS,MAPREDUCE,YARN) and fixVersion in ('"+"' ,
'".join([str(v).replace("-SNAPSHOT","") for v in versions])+"') and resolution
= Fixed", 'startAt':at, 'maxResults':count})
+ params = urllib.urlencode({'jql': "project in ('"+"' ,
'".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed",
'startAt':at, 'maxResults':count})
resp =
urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
data = json.loads(resp.read())
if (data.has_key('errorMessages')):
@@ -286,10 +310,8 @@ class JiraIter:
self.jiras.extend(data['issues'])
needaversion=False
- for j in versions:
-v=str(j).replace("-SNAPSHOT","")
-if v not in releaseVersion:
- needaversion=True
+ if v not in releaseVersion:
+needaversion=True
if needaversion is True:
for i in range(len(data['issues'])):
@@ -351,21 +373,29 @@ class Outputs:
self.writeKeyRaw(jira.getProject(), line)
def main():
- parser = OptionParser(usage="usage: %prog --version VERSION [--ver
[24/50] [abbrv] hadoop git commit: YARN-3026. Move application-specific container allocation logic from LeafQueue to FiCaSchedulerApp. Contributed by Wangda Tan
YARN-3026. Move application-specific container allocation logic from LeafQueue
to FiCaSchedulerApp. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83fe34ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83fe34ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83fe34ac
Branch: refs/heads/YARN-1197
Commit: 83fe34ac0896cee0918bbfad7bd51231e4aec39b
Parents: fc42fa8
Author: Jian He
Authored: Fri Jul 24 14:00:25 2015 -0700
Committer: Jian He
Committed: Fri Jul 24 14:00:25 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/resourcemanager/RMContextImpl.java | 3 +-
.../scheduler/ResourceLimits.java | 19 +-
.../scheduler/capacity/AbstractCSQueue.java | 27 +-
.../scheduler/capacity/CSAssignment.java| 12 +-
.../capacity/CapacityHeadroomProvider.java | 16 +-
.../scheduler/capacity/CapacityScheduler.java | 14 -
.../scheduler/capacity/LeafQueue.java | 833 +++
.../scheduler/capacity/ParentQueue.java | 16 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 721 +++-
.../capacity/TestApplicationLimits.java | 15 +-
.../capacity/TestCapacityScheduler.java | 3 +-
.../capacity/TestContainerAllocation.java | 85 +-
.../scheduler/capacity/TestLeafQueue.java | 191 +
.../scheduler/capacity/TestReservations.java| 111 +--
.../scheduler/capacity/TestUtils.java | 25 +-
16 files changed, 1048 insertions(+), 1046 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83fe34ac/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d1546b2..cf00fe5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -345,6 +345,9 @@ Release 2.8.0 - UNRELEASED
YARN-3844. Make hadoop-yarn-project Native code -Wall-clean (Alan Burlison
via Colin P. McCabe)
+YARN-3026. Move application-specific container allocation logic from
+LeafQueue to FiCaSchedulerApp. (Wangda Tan via jianhe)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83fe34ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 2f9209c..8cadc3b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -292,7 +292,8 @@ public class RMContextImpl implements RMContext {
activeServiceContext.setNMTokenSecretManager(nmTokenSecretManager);
}
- void setScheduler(ResourceScheduler scheduler) {
+ @VisibleForTesting
+ public void setScheduler(ResourceScheduler scheduler) {
activeServiceContext.setScheduler(scheduler);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/83fe34ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
index 8074794..c545e9e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceLimits.java
@@ -26,20 +26,25 @@ import org.apache.hadoop.yarn.util.res
[42/50] [abbrv] hadoop git commit: HADOOP-12245. References to misspelled REMAINING_QUATA in FileSystemShell.md. Contributed by Gabor Liptak.
HADOOP-12245. References to misspelled REMAINING_QUATA in FileSystemShell.md. Contributed by Gabor Liptak. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e21dde50 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e21dde50 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e21dde50 Branch: refs/heads/YARN-1197 Commit: e21dde501aa9323b7f34b4bc4ba9d282ec4f2707 Parents: 3572ebd Author: Akira Ajisaka Authored: Tue Jul 28 11:33:10 2015 +0900 Committer: Akira Ajisaka Committed: Tue Jul 28 11:33:10 2015 +0900 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop-common/src/site/markdown/FileSystemShell.md| 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e21dde50/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index baf39e3..aeaa5b9 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1017,6 +1017,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12239. StorageException complaining " no lease ID" when updating FolderLastModifiedTime in WASB. (Duo Xu via cnauroth) +HADOOP-12245. References to misspelled REMAINING_QUATA in +FileSystemShell.md. (Gabor Liptak via aajisaka) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/e21dde50/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md -- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md index 144cb73..fb89ca1 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md @@ -174,7 +174,7 @@ Usage: `hadoop fs -count [-q] [-h] [-v] ` Count the number of directories, files and bytes under the paths that match the specified file pattern. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME -The output columns with -count -q are: QUOTA, REMAINING\_QUATA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME +The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME The -h option shows sizes in human readable format.
[46/50] [abbrv] hadoop git commit: YARN-1449. AM-NM protocol changes to support container resizing. Contributed by Meng Ding & Wangda Tan)
YARN-1449. AM-NM protocol changes to support container resizing. Contributed by
Meng Ding & Wangda Tan)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3f2830d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3f2830d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3f2830d
Branch: refs/heads/YARN-1197
Commit: b3f2830df4b3b0a35d0b2d5542cae9dfecb2b397
Parents: 7758afd
Author: Jian He
Authored: Tue Jul 14 16:06:25 2015 -0700
Committer: Jian He
Committed: Tue Jul 28 13:53:32 2015 -0700
--
.../v2/app/launcher/TestContainerLauncher.java | 11 +
.../app/launcher/TestContainerLauncherImpl.java | 9 +
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/api/ContainerManagementProtocol.java | 30 ++-
.../IncreaseContainersResourceRequest.java | 75 ++
.../IncreaseContainersResourceResponse.java | 93 +++
.../proto/containermanagement_protocol.proto| 1 +
...ContainerManagementProtocolPBClientImpl.java | 20 ++
...ontainerManagementProtocolPBServiceImpl.java | 22 ++
...IncreaseContainersResourceRequestPBImpl.java | 170 +
...ncreaseContainersResourceResponsePBImpl.java | 241 +++
.../hadoop/yarn/TestContainerLaunchRPC.java | 8 +
.../yarn/TestContainerResourceIncreaseRPC.java | 162 +
.../java/org/apache/hadoop/yarn/TestRPC.java| 8 +
.../hadoop/yarn/api/TestPBImplRecords.java | 20 ++
.../containermanager/ContainerManagerImpl.java | 13 +
.../server/resourcemanager/NodeManager.java | 11 +-
.../resourcemanager/TestAMAuthorization.java| 8 +
.../TestApplicationMasterLauncher.java | 9 +
19 files changed, 910 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3f2830d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
index 41ee65d..6c3a4d6 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncher.java
@@ -30,6 +30,8 @@ import java.util.Map;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.atomic.AtomicInteger;
+import
org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
+import
org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.junit.Assert;
import org.apache.commons.logging.Log;
@@ -449,5 +451,14 @@ public class TestContainerLauncher {
"Dummy function cause"));
throw new IOException(e);
}
+
+@Override
+public IncreaseContainersResourceResponse increaseContainersResource(
+IncreaseContainersResourceRequest request) throws IOException,
+IOException {
+ Exception e = new Exception("Dummy function", new Exception(
+ "Dummy function cause"));
+ throw new IOException(e);
+}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3f2830d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
index 184f1b2..610448c 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/launcher/TestContainerLauncherImpl.java
@@ -46,6 +46,8 @@ import
org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher.EventType;
import org.a
[30/50] [abbrv] hadoop git commit: HADOOP-12237. releasedocmaker.py doesn't work behind a proxy (Tsuyoshi Ozawa via aw)
HADOOP-12237. releasedocmaker.py doesn't work behind a proxy (Tsuyoshi Ozawa
via aw)
(cherry picked from commit b41fe3111ae37478cbace2a07e6ac35a676ef978)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adcf5dd9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adcf5dd9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adcf5dd9
Branch: refs/heads/YARN-1197
Commit: adcf5dd94052481f66deaf402ac4ace1ffc06f49
Parents: d769783
Author: Allen Wittenauer
Authored: Mon Jul 20 09:47:46 2015 -0700
Committer: Andrew Wang
Committed: Fri Jul 24 18:31:48 2015 -0700
--
dev-support/releasedocmaker.py | 11 ---
1 file changed, 8 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adcf5dd9/dev-support/releasedocmaker.py
--
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index 409d8e3..d2e5dda 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -24,6 +24,7 @@ import os
import re
import sys
import urllib
+import urllib2
try:
import json
except ImportError:
@@ -125,7 +126,7 @@ class GetVersions:
versions.sort()
print "Looking for %s through %s"%(versions[0],versions[-1])
for p in projects:
- resp =
urllib.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
+ resp =
urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/project/%s/versions"%p)
data = json.loads(resp.read())
for d in data:
if d['name'][0].isdigit and versions[0] <= d['name'] and d['name'] <=
versions[-1]:
@@ -288,7 +289,7 @@ class JiraIter:
self.projects = projects
v=str(version).replace("-SNAPSHOT","")
-resp = urllib.urlopen("https://issues.apache.org/jira/rest/api/2/field";)
+resp = urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/field";)
data = json.loads(resp.read())
self.fieldIdMap = {}
@@ -301,7 +302,7 @@ class JiraIter:
count=100
while (at < end):
params = urllib.urlencode({'jql': "project in ('"+"' ,
'".join(projects)+"') and fixVersion in ('"+v+"') and resolution = Fixed",
'startAt':at, 'maxResults':count})
- resp =
urllib.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
+ resp =
urllib2.urlopen("https://issues.apache.org/jira/rest/api/2/search?%s"%params)
data = json.loads(resp.read())
if (data.has_key('errorMessages')):
raise Exception(data['errorMessages'])
@@ -407,6 +408,10 @@ def main():
if (len(options.versions) <= 0):
parser.error("At least one version needs to be supplied")
+ proxy = urllib2.ProxyHandler()
+ opener = urllib2.build_opener(proxy)
+ urllib2.install_opener(opener)
+
projects = options.projects
if (options.range is True):
[36/50] [abbrv] hadoop git commit: HDFS-8810. Correct assertions in TestDFSInotifyEventInputStream class. Contributed by Surendra Singh Lilhore.
HDFS-8810. Correct assertions in TestDFSInotifyEventInputStream class.
Contributed by Surendra Singh Lilhore.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1df78688
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1df78688
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1df78688
Branch: refs/heads/YARN-1197
Commit: 1df78688c69476f89d16f93bc74a4f05d0b1a3da
Parents: 42d4e0a
Author: Akira Ajisaka
Authored: Mon Jul 27 13:17:24 2015 +0900
Committer: Akira Ajisaka
Committed: Mon Jul 27 13:17:24 2015 +0900
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1df78688/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3614e01..1ddf7da 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1081,6 +1081,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8773. Few FSNamesystem metrics are not documented in the Metrics page.
(Rakesh R via cnauroth)
+HDFS-8810. Correct assertions in TestDFSInotifyEventInputStream class.
+(Surendra Singh Lilhore via aajisaka)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1df78688/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
index 65569d0..e7bbcac 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
@@ -164,7 +164,7 @@ public class TestDFSInotifyEventInputStream {
Event.RenameEvent re2 = (Event.RenameEvent) batch.getEvents()[0];
Assert.assertTrue(re2.getDstPath().equals("/file2"));
Assert.assertTrue(re2.getSrcPath().equals("/file4"));
- Assert.assertTrue(re.getTimestamp() > 0);
+ Assert.assertTrue(re2.getTimestamp() > 0);
LOG.info(re2.toString());
// AddOp with overwrite
@@ -378,7 +378,7 @@ public class TestDFSInotifyEventInputStream {
Event.RenameEvent re3 = (Event.RenameEvent) batch.getEvents()[0];
Assert.assertTrue(re3.getDstPath().equals("/dir/file5"));
Assert.assertTrue(re3.getSrcPath().equals("/file5"));
- Assert.assertTrue(re.getTimestamp() > 0);
+ Assert.assertTrue(re3.getTimestamp() > 0);
LOG.info(re3.toString());
// TruncateOp
[06/50] [abbrv] hadoop git commit: YARN-2019. Retrospect on decision of making RM crashed if any exception throw in ZKRMStateStore. Contributed by Jian He.
YARN-2019. Retrospect on decision of making RM crashed if any exception throw
in ZKRMStateStore. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee98d635
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee98d635
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee98d635
Branch: refs/heads/YARN-1197
Commit: ee98d6354bbbcd0832d3e539ee097f837e5d0e31
Parents: e91ccfa
Author: Junping Du
Authored: Wed Jul 22 17:52:35 2015 -0700
Committer: Junping Du
Committed: Wed Jul 22 17:52:35 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 11 +++
.../src/main/resources/yarn-default.xml | 16
.../resourcemanager/recovery/RMStateStore.java | 9 +++--
4 files changed, 37 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee98d635/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a5fd4e7..93962f1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -144,6 +144,9 @@ Release 2.8.0 - UNRELEASED
YARN-2003. Support for Application priority : Changes in RM and Capacity
Scheduler. (Sunil G via wangda)
+YARN-2019. Retrospect on decision of making RM crashed if any exception
throw
+in ZKRMStateStore. (Jian He via junping_du)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee98d635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 060635f..9832729 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -401,6 +401,11 @@ public class YarnConfiguration extends Configuration {
public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled";
public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false;
+ public static final String YARN_FAIL_FAST = YARN_PREFIX + "fail-fast";
+ public static final boolean DEFAULT_YARN_FAIL_FAST = true;
+
+ public static final String RM_FAIL_FAST = RM_PREFIX + "fail-fast";
+
@Private
public static final String RM_WORK_PRESERVING_RECOVERY_ENABLED = RM_PREFIX
+ "work-preserving-recovery.enabled";
@@ -2018,6 +2023,12 @@ public class YarnConfiguration extends Configuration {
YARN_HTTP_POLICY_DEFAULT));
}
+ public static boolean shouldRMFailFast(Configuration conf) {
+return conf.getBoolean(YarnConfiguration.RM_FAIL_FAST,
+conf.getBoolean(YarnConfiguration.YARN_FAIL_FAST,
+YarnConfiguration.DEFAULT_YARN_FAIL_FAST));
+ }
+
@Private
public static String getClusterId(Configuration conf) {
String clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee98d635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index d586f51..8b3a3af 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -324,6 +324,22 @@
+Should RM fail fast if it encounters any errors. By defalt, it
+ points to ${yarn.fail-fast}. Errors include:
+ 1) exceptions when state-store write/read operations fails.
+
+yarn.resourcemanager.fail-fast
+${yarn.fail-fast}
+
+
+
+Should YARN fail fast if it encounters any errors.
+
+yarn.fail-fast
+true
+
+
+
Enable RM work preserving recovery. This configuration is
private
to YARN for experimenting the feature.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee98d635/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/se
[10/50] [abbrv] hadoop git commit: HADOOP-12189. Improve CallQueueManager#swapQueue to make queue elements drop nearly impossible. Contributed by Zhihai Xu.
HADOOP-12189. Improve CallQueueManager#swapQueue to make queue elements drop
nearly impossible. Contributed by Zhihai Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6736a1ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6736a1ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6736a1ab
Branch: refs/heads/YARN-1197
Commit: 6736a1ab7033523ed5f304fdfed46d7f348665b4
Parents: 813cf89
Author: Andrew Wang
Authored: Thu Jul 23 14:42:35 2015 -0700
Committer: Andrew Wang
Committed: Thu Jul 23 14:42:35 2015 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../org/apache/hadoop/ipc/CallQueueManager.java | 27 +---
.../apache/hadoop/ipc/TestCallQueueManager.java | 6 ++---
3 files changed, 24 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6736a1ab/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index f1a3bc9..6c18add 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -719,6 +719,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12161. Add getStoragePolicy API to the FileSystem interface.
(Brahma Reddy Battula via Arpit Agarwal)
+HADOOP-12189. Improve CallQueueManager#swapQueue to make queue elements
+drop nearly impossible. (Zhihai Xu via wang)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6736a1ab/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
index 1568bd6..c10f839 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/CallQueueManager.java
@@ -32,11 +32,15 @@ import org.apache.hadoop.conf.Configuration;
*/
public class CallQueueManager {
public static final Log LOG = LogFactory.getLog(CallQueueManager.class);
+ // Number of checkpoints for empty queue.
+ private static final int CHECKPOINT_NUM = 20;
+ // Interval to check empty queue.
+ private static final long CHECKPOINT_INTERVAL_MS = 10;
@SuppressWarnings("unchecked")
static Class> convertQueueClass(
- Class queneClass, Class elementClass) {
-return (Class>)queneClass;
+ Class queueClass, Class elementClass) {
+return (Class>)queueClass;
}
private final boolean clientBackOffEnabled;
@@ -159,18 +163,23 @@ public class CallQueueManager {
}
/**
- * Checks if queue is empty by checking at two points in time.
+ * Checks if queue is empty by checking at CHECKPOINT_NUM points with
+ * CHECKPOINT_INTERVAL_MS interval.
* This doesn't mean the queue might not fill up at some point later, but
* it should decrease the probability that we lose a call this way.
*/
private boolean queueIsReallyEmpty(BlockingQueue q) {
-boolean wasEmpty = q.isEmpty();
-try {
- Thread.sleep(10);
-} catch (InterruptedException ie) {
- return false;
+for (int i = 0; i < CHECKPOINT_NUM; i++) {
+ try {
+Thread.sleep(CHECKPOINT_INTERVAL_MS);
+ } catch (InterruptedException ie) {
+return false;
+ }
+ if (!q.isEmpty()) {
+return false;
+ }
}
-return q.isEmpty() && wasEmpty;
+return true;
}
private String stringRepr(Object o) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6736a1ab/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
index 6e1838e..51a9750 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestCallQueueManager.java
@@ -165,7 +165,7 @@ public class TestCallQueueManager {
HashMap threads = new HashMap();
// Create putters and takers
-for (int i=0; i < 50; i++) {
+for (int i=0; i < 1000; i++) {
Putter p = new Putter(manager
[21/50] [abbrv] hadoop git commit: HADOOP-12170. hadoop-common's JNIFlags.cmake is redundant and can be removed (Alan Burlison via Colin P. McCabe)
HADOOP-12170. hadoop-common's JNIFlags.cmake is redundant and can be removed
(Alan Burlison via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4b0c744
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4b0c744
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4b0c744
Branch: refs/heads/YARN-1197
Commit: e4b0c74434b82c25256a59b03d62b1a66bb8ac69
Parents: d19d187
Author: Colin Patrick Mccabe
Authored: Fri Jul 24 13:03:31 2015 -0700
Committer: Colin Patrick Mccabe
Committed: Fri Jul 24 13:03:31 2015 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop-common/src/JNIFlags.cmake| 124 ---
2 files changed, 3 insertions(+), 124 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4b0c744/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index d6d43f2..0da6194 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -727,6 +727,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12259. Utility to Dynamic port allocation (brahmareddy via rkanter)
+HADOOP-12170. hadoop-common's JNIFlags.cmake is redundant and can be
+removed (Alan Burlison via Colin P. McCabe)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4b0c744/hadoop-common-project/hadoop-common/src/JNIFlags.cmake
--
diff --git a/hadoop-common-project/hadoop-common/src/JNIFlags.cmake
b/hadoop-common-project/hadoop-common/src/JNIFlags.cmake
deleted file mode 100644
index c558fe8..000
--- a/hadoop-common-project/hadoop-common/src/JNIFlags.cmake
+++ /dev/null
@@ -1,124 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-
-# If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit.
-# This variable is set by maven.
-if (JVM_ARCH_DATA_MODEL EQUAL 32)
-# Force 32-bit code generation on amd64/x86_64, ppc64, sparc64
-if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_SYSTEM_PROCESSOR MATCHES ".*64")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")
-set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32")
-endif ()
-if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR
STREQUAL "amd64")
-# Set CMAKE_SYSTEM_PROCESSOR to ensure that find_package(JNI) will use
-# the 32-bit version of libjvm.so.
-set(CMAKE_SYSTEM_PROCESSOR "i686")
-endif ()
-endif (JVM_ARCH_DATA_MODEL EQUAL 32)
-
-# Determine float ABI of JVM on ARM Linux
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm" AND CMAKE_SYSTEM_NAME STREQUAL
"Linux")
-find_program(READELF readelf)
-if (READELF MATCHES "NOTFOUND")
-message(WARNING "readelf not found; JVM float ABI detection disabled")
-else (READELF MATCHES "NOTFOUND")
-execute_process(
-COMMAND ${READELF} -A ${JAVA_JVM_LIBRARY}
-OUTPUT_VARIABLE JVM_ELF_ARCH
-ERROR_QUIET)
-if (NOT JVM_ELF_ARCH MATCHES "Tag_ABI_VFP_args: VFP registers")
-message("Soft-float JVM detected")
-
-# Test compilation with -mfloat-abi=softfp using an arbitrary libc
function
-# (typically fails with "fatal error: bits/predefs.h: No such file
or directory"
-# if soft-float dev libraries are not installed)
-include(CMakePushCheckState)
-cmake_push_check_state()
-set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS}
-mfloat-abi=softfp")
-include(CheckSymbolExists)
-check_symbol_exists(exit stdlib.h SOFTFP_AVAILABLE)
-if (NOT SOFTFP_AVAILABLE)
-message(FATAL_ERROR "Soft-flo
[13/50] [abbrv] hadoop git commit: YARN-3969. Allow jobs to be submitted to reservation that is active but does not have any allocations. (subru via curino)
YARN-3969. Allow jobs to be submitted to reservation that is active but does
not have any allocations. (subru via curino)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0fcb4a8c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0fcb4a8c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0fcb4a8c
Branch: refs/heads/YARN-1197
Commit: 0fcb4a8cf2add3f112907ff4e833e2f04947b53e
Parents: 206d493
Author: carlo curino
Authored: Thu Jul 23 19:33:59 2015 -0700
Committer: carlo curino
Committed: Thu Jul 23 19:33:59 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../scheduler/capacity/ReservationQueue.java| 4 ---
.../capacity/TestReservationQueue.java | 26 +++-
3 files changed, 17 insertions(+), 16 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fcb4a8c/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f23853b..8bc9e4c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -864,6 +864,9 @@ Release 2.7.1 - 2015-07-06
YARN-3850. NM fails to read files from full disks which can lead to
container logs being lost and other issues (Varun Saxena via jlowe)
+YARN-3969. Allow jobs to be submitted to reservation that is active
+but does not have any allocations. (subru via curino)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fcb4a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
index 4790cc7..976cf8c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
@@ -39,12 +39,9 @@ public class ReservationQueue extends LeafQueue {
private PlanQueue parent;
- private int maxSystemApps;
-
public ReservationQueue(CapacitySchedulerContext cs, String queueName,
PlanQueue parent) throws IOException {
super(cs, queueName, parent, null);
-maxSystemApps = cs.getConfiguration().getMaximumSystemApplications();
// the following parameters are common to all reservation in the plan
updateQuotas(parent.getUserLimitForReservation(),
parent.getUserLimitFactor(),
@@ -89,7 +86,6 @@ public class ReservationQueue extends LeafQueue {
}
setCapacity(capacity);
setAbsoluteCapacity(getParent().getAbsoluteCapacity() * getCapacity());
-setMaxApplications((int) (maxSystemApps * getAbsoluteCapacity()));
// note: we currently set maxCapacity to capacity
// this might be revised later
setMaxCapacity(entitlement.getMaxCapacity());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0fcb4a8c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
index 4e6c73d..e23e93c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservationQueue.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.serv
[19/50] [abbrv] hadoop git commit: HDFS-8735. Inotify: All events classes should implement toString() API. Contributed by Surendra Singh Lilhore.
HDFS-8735. Inotify: All events classes should implement toString() API.
Contributed by Surendra Singh Lilhore.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f8f60918
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f8f60918
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f8f60918
Branch: refs/heads/YARN-1197
Commit: f8f60918230dd466ae8dda1fbc28878e19273232
Parents: fbd6063
Author: Akira Ajisaka
Authored: Sat Jul 25 02:56:55 2015 +0900
Committer: Akira Ajisaka
Committed: Sat Jul 25 02:56:55 2015 +0900
--
.../org/apache/hadoop/hdfs/inotify/Event.java | 95
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/TestDFSInotifyEventInputStream.java| 26 ++
3 files changed, 124 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f8f60918/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
index dee17a9..6f2b5e2 100644
---
a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
+++
b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/inotify/Event.java
@@ -51,6 +51,7 @@ public abstract class Event {
/**
* Sent when a file is closed after append or create.
*/
+ @InterfaceAudience.Public
public static class CloseEvent extends Event {
private String path;
private long fileSize;
@@ -81,11 +82,20 @@ public abstract class Event {
public long getTimestamp() {
return timestamp;
}
+
+@Override
+@InterfaceStability.Unstable
+public String toString() {
+ return "CloseEvent [path=" + path + ", fileSize=" + fileSize
+ + ", timestamp=" + timestamp + "]";
+}
+
}
/**
* Sent when a new file is created (including overwrite).
*/
+ @InterfaceAudience.Public
public static class CreateEvent extends Event {
public static enum INodeType {
@@ -232,6 +242,25 @@ public abstract class Event {
public long getDefaultBlockSize() {
return defaultBlockSize;
}
+
+@Override
+@InterfaceStability.Unstable
+public String toString() {
+ StringBuilder content = new StringBuilder();
+ content.append("CreateEvent [INodeType=" + iNodeType + ", path=" + path
+ + ", ctime=" + ctime + ", replication=" + replication
+ + ", ownerName=" + ownerName + ", groupName=" + groupName
+ + ", perms=" + perms + ", ");
+
+ if (symlinkTarget != null) {
+content.append("symlinkTarget=" + symlinkTarget + ", ");
+ }
+
+ content.append("overwrite=" + overwrite + ", defaultBlockSize="
+ + defaultBlockSize + "]");
+ return content.toString();
+}
+
}
/**
@@ -242,6 +271,7 @@ public abstract class Event {
* metadataType of the MetadataUpdateEvent will be null or will have their
default
* values.
*/
+ @InterfaceAudience.Public
public static class MetadataUpdateEvent extends Event {
public static enum MetadataType {
@@ -400,11 +430,45 @@ public abstract class Event {
return xAttrsRemoved;
}
+@Override
+@InterfaceStability.Unstable
+public String toString() {
+ StringBuilder content = new StringBuilder();
+ content.append("MetadataUpdateEvent [path=" + path + ", metadataType="
+ + metadataType);
+ switch (metadataType) {
+ case TIMES:
+content.append(", mtime=" + mtime + ", atime=" + atime);
+break;
+ case REPLICATION:
+content.append(", replication=" + replication);
+break;
+ case OWNER:
+content.append(", ownerName=" + ownerName
++ ", groupName=" + groupName);
+break;
+ case PERMS:
+content.append(", perms=" + perms);
+break;
+ case ACLS:
+content.append(", acls=" + acls);
+break;
+ case XATTRS:
+content.append(", xAttrs=" + xAttrs + ", xAttrsRemoved="
++ xAttrsRemoved);
+break;
+ default:
+break;
+ }
+ content.append(']');
+ return content.toString();
+}
}
/**
* Sent when a file, directory, or symlink is renamed.
*/
+ @InterfaceAudience.Public
public static class RenameEvent extends Event {
private String srcPath;
private String dstPath;
@@ -456,11 +520,20 @@ public abstract class Event {
public long getTimestamp() {
return timestamp;
}
+
+@Override
+@InterfaceStability.Unstable
+public Stri
[32/50] [abbrv] hadoop git commit: YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations. (Jonathan Yaniv and Ishai Menache via curino)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/156f24ea/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java
deleted file mode 100644
index de94dcd..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestGreedyReservationAgent.java
+++ /dev/null
@@ -1,604 +0,0 @@
-/***
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-
***/
-package org.apache.hadoop.yarn.server.resourcemanager.reservation;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.mock;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Random;
-
-import org.apache.hadoop.yarn.api.records.ReservationDefinition;
-import org.apache.hadoop.yarn.api.records.ReservationId;
-import org.apache.hadoop.yarn.api.records.ReservationRequest;
-import org.apache.hadoop.yarn.api.records.ReservationRequestInterpreter;
-import org.apache.hadoop.yarn.api.records.ReservationRequests;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.impl.pb.ReservationDefinitionPBImpl;
-import org.apache.hadoop.yarn.api.records.impl.pb.ReservationRequestsPBImpl;
-import
org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
-import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.Resources;
-import org.junit.Before;
-import org.junit.Test;
-import org.mortbay.log.Log;
-
-public class TestGreedyReservationAgent {
-
- ReservationAgent agent;
- InMemoryPlan plan;
- Resource minAlloc = Resource.newInstance(1024, 1);
- ResourceCalculator res = new DefaultResourceCalculator();
- Resource maxAlloc = Resource.newInstance(1024 * 8, 8);
- Random rand = new Random();
- long step;
-
- @Before
- public void setup() throws Exception {
-
-long seed = rand.nextLong();
-rand.setSeed(seed);
-Log.info("Running with seed: " + seed);
-
-// setting completely loose quotas
-long timeWindow = 100L;
-Resource clusterCapacity = Resource.newInstance(100 * 1024, 100);
-step = 1000L;
-ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
-String reservationQ = testUtil.getFullReservationQueueName();
-
-float instConstraint = 100;
-float avgConstraint = 100;
-
-ReservationSchedulerConfiguration conf =
-ReservationSystemTestUtil.createConf(reservationQ, timeWindow,
-instConstraint, avgConstraint);
-CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
-policy.init(reservationQ, conf);
-agent = new GreedyReservationAgent();
-
-QueueMetrics queueMetrics = mock(QueueMetrics.class);
-
-plan = new InMemoryPlan(queueMetrics, policy, agent, clusterCapacity, step,
-res, minAlloc, maxAlloc, "dedicated", null, true);
- }
-
- @SuppressWarnings("javadoc")
- @Test
- public void testSimple() throws PlanningException {
-
-prepareBasicPlan();
-
-// create
[22/50] [abbrv] hadoop git commit: YARN-3969. Updating CHANGES.txt to reflect the correct set of branches where this is committed
YARN-3969. Updating CHANGES.txt to reflect the correct set of branches where this is committed Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fc42fa8a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fc42fa8a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fc42fa8a Branch: refs/heads/YARN-1197 Commit: fc42fa8ae3bc9d6d055090a7bb5e6f0c5972fcff Parents: e4b0c74 Author: carlo curino Authored: Fri Jul 24 13:38:44 2015 -0700 Committer: carlo curino Committed: Fri Jul 24 13:38:44 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 6 -- 1 file changed, 4 insertions(+), 2 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/fc42fa8a/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 44e5510..d1546b2 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -710,6 +710,10 @@ Release 2.7.2 - UNRELEASED YARN-3878. AsyncDispatcher can hang while stopping if it is configured for draining events on stop. (Varun Saxena via jianhe) +YARN-3969. Allow jobs to be submitted to reservation that is active +but does not have any allocations. (subru via curino) + + Release 2.7.1 - 2015-07-06 INCOMPATIBLE CHANGES @@ -870,8 +874,6 @@ Release 2.7.1 - 2015-07-06 YARN-3850. NM fails to read files from full disks which can lead to container logs being lost and other issues (Varun Saxena via jlowe) -YARN-3969. Allow jobs to be submitted to reservation that is active -but does not have any allocations. (subru via curino) Release 2.7.0 - 2015-04-20
[17/50] [abbrv] hadoop git commit: HADOOP-12259. Utility to Dynamic port allocation (brahmareddy via rkanter)
HADOOP-12259. Utility to Dynamic port allocation (brahmareddy via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee233ec9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee233ec9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee233ec9
Branch: refs/heads/YARN-1197
Commit: ee233ec95ce8cfc8309d3adc072d926cd85eba08
Parents: 0fcb4a8
Author: Robert Kanter
Authored: Fri Jul 24 09:41:53 2015 -0700
Committer: Robert Kanter
Committed: Fri Jul 24 09:41:53 2015 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../org/apache/hadoop/net/ServerSocketUtil.java | 63
2 files changed, 65 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee233ec9/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 56edcac..d6d43f2 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -725,6 +725,8 @@ Release 2.8.0 - UNRELEASED
HADOOP-12189. Improve CallQueueManager#swapQueue to make queue elements
drop nearly impossible. (Zhihai Xu via wang)
+HADOOP-12259. Utility to Dynamic port allocation (brahmareddy via rkanter)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee233ec9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
new file mode 100644
index 000..0ce835f
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/ServerSocketUtil.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.net;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class ServerSocketUtil {
+
+ private static final Log LOG = LogFactory.getLog(ServerSocketUtil.class);
+
+ /**
+ * Port scan & allocate is how most other apps find ports
+ *
+ * @param port given port
+ * @param retries number of retires
+ * @return
+ * @throws IOException
+ */
+ public static int getPort(int port, int retries) throws IOException {
+Random rand = new Random();
+int tryPort = port;
+int tries = 0;
+while (true) {
+ if (tries > 0) {
+tryPort = port + rand.nextInt(65535 - port);
+ }
+ LOG.info("Using port " + tryPort);
+ try (ServerSocket s = new ServerSocket(tryPort)) {
+return tryPort;
+ } catch (IOException e) {
+tries++;
+if (tries >= retries) {
+ LOG.info("Port is already in use; giving up");
+ throw e;
+} else {
+ LOG.info("Port is already in use; trying again");
+}
+ }
+}
+ }
+
+}
[11/50] [abbrv] hadoop git commit: YARN-3900. Protobuf layout of yarn_security_token causes errors in other protos that include it (adhoot via rkanter)
YARN-3900. Protobuf layout of yarn_security_token causes errors in other protos
that include it (adhoot via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d3026e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d3026e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d3026e7
Branch: refs/heads/YARN-1197
Commit: 1d3026e7b3cf2f3a8a544b66ff14783cc590bdac
Parents: 6736a1a
Author: Robert Kanter
Authored: Thu Jul 23 14:42:49 2015 -0700
Committer: Robert Kanter
Committed: Thu Jul 23 14:46:54 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 2 +-
.../main/proto/server/yarn_security_token.proto | 70
.../src/main/proto/yarn_security_token.proto| 70
.../pom.xml | 2 +-
.../hadoop-yarn-server-resourcemanager/pom.xml | 2 +-
.../resourcemanager/recovery/TestProtos.java| 36 ++
7 files changed, 112 insertions(+), 73 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d3026e7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9416cd6..3d41ba7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -666,6 +666,9 @@ Release 2.8.0 - UNRELEASED
YARN-3941. Proportional Preemption policy should try to avoid sending
duplicate PREEMPT_CONTAINER event to scheduler. (Sunil G via wangda)
+YARN-3900. Protobuf layout of yarn_security_token causes errors in other
protos
+that include it (adhoot via rkanter)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d3026e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
--
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index 602fcd7..3b47cdd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -253,7 +253,7 @@
${basedir}/src/main/proto
-${basedir}/src/main/proto/server
+${basedir}/src/main/proto
yarn_security_token.proto
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d3026e7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto
deleted file mode 100644
index 339e99e..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/proto/server/yarn_security_token.proto
+++ /dev/null
@@ -1,70 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.hadoop.yarn.proto";
-option java_outer_classname = "YarnSecurityTokenProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.yarn;
-
-import "yarn_protos.proto";
-
-// None of the following records are supposed to be exposed to users.
-
-message NMTokenIdentifierProto {
- optional ApplicationAttemptIdProto appAttemptId = 1;
- optional NodeIdProto nodeId = 2;
- optional string appSubmitter = 3;
- optional int32 keyId = 4 [default = -1];
-}
-
-message AMRMTokenIdentifierProto {
- optional ApplicationAttemptIdProto appAttemptId = 1;
- optional int32 keyId = 2 [default = -1];
-}
-
-message ContainerTokenIdentifierProto {
- optional ContainerIdProto containerId = 1;
- optional string nmHostAddr = 2;
- option
[25/50] [abbrv] hadoop git commit: YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container log files from full disks. Contributed by zhihai xu
YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container log
files from full disks. Contributed by zhihai xu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff9c13e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff9c13e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff9c13e0
Branch: refs/heads/YARN-1197
Commit: ff9c13e0a739bb13115167dc661b6a16b2ed2c04
Parents: 83fe34a
Author: Jason Lowe
Authored: Fri Jul 24 22:14:39 2015 +
Committer: Jason Lowe
Committed: Fri Jul 24 22:14:39 2015 +
--
hadoop-yarn-project/CHANGES.txt | 2 +
.../nodemanager/LocalDirsHandlerService.java| 35 +-
.../webapp/TestContainerLogsPage.java | 48
3 files changed, 83 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff9c13e0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cf00fe5..c295784 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -716,6 +716,8 @@ Release 2.7.2 - UNRELEASED
YARN-3969. Allow jobs to be submitted to reservation that is active
but does not have any allocations. (subru via curino)
+YARN-3925. ContainerLogsUtils#getContainerLogFile fails to read container
+log files from full disks. (zhihai xu via jlowe)
Release 2.7.1 - 2015-07-06
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff9c13e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index 0a61035..6709c90 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
@@ -31,6 +32,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -467,6 +469,35 @@ public class LocalDirsHandlerService extends
AbstractService {
return disksTurnedGood;
}
+ private Path getPathToRead(String pathStr, List dirs)
+ throws IOException {
+// remove the leading slash from the path (to make sure that the uri
+// resolution results in a valid path on the dir being checked)
+if (pathStr.startsWith("/")) {
+ pathStr = pathStr.substring(1);
+}
+
+FileSystem localFS = FileSystem.getLocal(getConfig());
+for (String dir : dirs) {
+ try {
+Path tmpDir = new Path(dir);
+File tmpFile = tmpDir.isAbsolute()
+? new File(localFS.makeQualified(tmpDir).toUri())
+: new File(dir);
+Path file = new Path(tmpFile.getPath(), pathStr);
+if (localFS.exists(file)) {
+ return file;
+}
+ } catch (IOException ie) {
+// ignore
+LOG.warn("Failed to find " + pathStr + " at " + dir, ie);
+ }
+}
+
+throw new IOException("Could not find " + pathStr + " in any of" +
+" the directories");
+ }
+
public Path getLocalPathForWrite(String pathStr) throws IOException {
return localDirsAllocator.getLocalPathForWrite(pathStr, getConfig());
}
@@ -484,9 +515,9 @@ public class LocalDirsHandlerService extends
AbstractService {
}
public Path getLogPathToRead(String pathStr) throws IOException {
-return logDirsAllocator.getLocalPathToRead(pathStr, getConfig());
+return getPathToRead(pathStr, getLogDirsForRead());
}
-
+
public static String[] validatePaths(String[] paths) {
ArrayList validPaths = new ArrayList();
for (int i = 0; i < paths.length; ++i) {
http://git-wip-us.apache.or
[38/50] [abbrv] hadoop git commit: YARN-3852. Add docker container support to container-executor. Contributed by Abin Shahab.
YARN-3852. Add docker container support to container-executor. Contributed by
Abin Shahab.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f36835ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f36835ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f36835ff
Branch: refs/heads/YARN-1197
Commit: f36835ff9b878fa20fe58a30f9d1e8c47702d6d2
Parents: 2196e39
Author: Varun Vasudev
Authored: Mon Jul 27 10:12:30 2015 -0700
Committer: Varun Vasudev
Committed: Mon Jul 27 10:14:51 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../container-executor/impl/configuration.c | 17 +-
.../container-executor/impl/configuration.h | 2 +
.../impl/container-executor.c | 417 ---
.../impl/container-executor.h | 25 +-
.../main/native/container-executor/impl/main.c | 97 -
6 files changed, 480 insertions(+), 81 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f36835ff/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3b7d8a8..4e54aea 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -150,6 +150,9 @@ Release 2.8.0 - UNRELEASED
YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations.
(Jonathan Yaniv and Ishai Menache via curino)
+YARN-3852. Add docker container support to container-executor
+(Abin Shahab via vvasudev)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f36835ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index eaa1f19..2825367 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -291,27 +291,23 @@ char ** get_values(const char * key) {
return extract_values(value);
}
-/**
- * Extracts array of values from the '%' separated list of values.
- */
-char ** extract_values(char *value) {
+char ** extract_values_delim(char *value, const char *delim) {
char ** toPass = NULL;
char *tempTok = NULL;
char *tempstr = NULL;
int size = 0;
int toPassSize = MAX_SIZE;
-
//first allocate any array of 10
if(value != NULL) {
toPass = (char **) malloc(sizeof(char *) * toPassSize);
-tempTok = strtok_r((char *)value, "%", &tempstr);
+tempTok = strtok_r((char *)value, delim, &tempstr);
while (tempTok != NULL) {
toPass[size++] = tempTok;
if(size == toPassSize) {
toPassSize += MAX_SIZE;
toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize));
}
- tempTok = strtok_r(NULL, "%", &tempstr);
+ tempTok = strtok_r(NULL, delim, &tempstr);
}
}
if (toPass != NULL) {
@@ -320,6 +316,13 @@ char ** extract_values(char *value) {
return toPass;
}
+/**
+ * Extracts array of values from the '%' separated list of values.
+ */
+char ** extract_values(char *value) {
+ extract_values_delim(value, "%");
+}
+
// free an entry set of values
void free_values(char** values) {
if (*values != NULL) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f36835ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
index 133e67b..390a5b5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
@@ -46,6 +46,8 @@ char ** get_values(con
[37/50] [abbrv] hadoop git commit: HDFS-8785. TestDistributedFileSystem is failing in trunk. Contributed by Xiaoyu Yao.
HDFS-8785. TestDistributedFileSystem is failing in trunk. Contributed by Xiaoyu
Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2196e39e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2196e39e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2196e39e
Branch: refs/heads/YARN-1197
Commit: 2196e39e142b0f8d1944805db2bfacd4e3244625
Parents: 1df7868
Author: Xiaoyu Yao
Authored: Mon Jul 27 07:28:41 2015 -0700
Committer: Xiaoyu Yao
Committed: Mon Jul 27 07:28:41 2015 -0700
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt| 2 ++
.../apache/hadoop/hdfs/TestDistributedFileSystem.java | 13 -
2 files changed, 10 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2196e39e/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1ddf7da..cc2a833 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1084,6 +1084,8 @@ Release 2.8.0 - UNRELEASED
HDFS-8810. Correct assertions in TestDFSInotifyEventInputStream class.
(Surendra Singh Lilhore via aajisaka)
+HDFS-8785. TestDistributedFileSystem is failing in trunk. (Xiaoyu Yao)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2196e39e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
index 0b77210..6012c5d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDistributedFileSystem.java
@@ -1189,19 +1189,22 @@ public class TestDistributedFileSystem {
try {
cluster.waitActive();
DistributedFileSystem dfs = cluster.getFileSystem();
- // Write 1 MB to a dummy socket to ensure the write times out
+ // Write 10 MB to a dummy socket to ensure the write times out
ServerSocket socket = new ServerSocket(0);
Peer peer = dfs.getClient().newConnectedPeer(
(InetSocketAddress) socket.getLocalSocketAddress(), null, null);
long start = Time.now();
try {
-byte[] buf = new byte[1024 * 1024];
+byte[] buf = new byte[10 * 1024 * 1024];
peer.getOutputStream().write(buf);
-Assert.fail("write should timeout");
+long delta = Time.now() - start;
+Assert.fail("write finish in " + delta + " ms" + "but should
timedout");
} catch (SocketTimeoutException ste) {
long delta = Time.now() - start;
-Assert.assertTrue("write timedout too soon", delta >= timeout * 0.9);
-Assert.assertTrue("write timedout too late", delta <= timeout * 1.1);
+Assert.assertTrue("write timedout too soon in " + delta + " ms",
+delta >= timeout * 0.9);
+Assert.assertTrue("write timedout too late in " + delta + " ms",
+delta <= timeout * 1.2);
} catch (Throwable t) {
Assert.fail("wrong exception:" + t);
}
[14/50] [abbrv] hadoop git commit: HDFS-6682. Add a metric to expose the timestamp of the oldest under-replicated block. (aajisaka)
HDFS-6682. Add a metric to expose the timestamp of the oldest under-replicated
block. (aajisaka)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02c01815
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02c01815
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02c01815
Branch: refs/heads/YARN-1197
Commit: 02c01815eca656814febcdaca6115e5f53b9c746
Parents: ab3197c
Author: Akira Ajisaka
Authored: Fri Jul 24 11:37:23 2015 +0900
Committer: Akira Ajisaka
Committed: Fri Jul 24 11:37:23 2015 +0900
--
.../hadoop-common/src/site/markdown/Metrics.md | 1 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../server/blockmanagement/BlockManager.java| 4 ++
.../blockmanagement/UnderReplicatedBlocks.java | 33 --
.../hdfs/server/namenode/FSNamesystem.java | 9 +++-
.../TestUnderReplicatedBlocks.java | 48
6 files changed, 93 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02c01815/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
--
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
index 646cda5..2e6c095 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -201,6 +201,7 @@ Each metrics record contains tags such as HAState and
Hostname as additional inf
| Name | Description |
|: |: |
| `MissingBlocks` | Current number of missing blocks |
+| `TimeOfTheOldestBlockToBeReplicated` | The timestamp of the oldest block to
be replicated. If there are no under-replicated or corrupt blocks, return 0. |
| `ExpiredHeartbeats` | Total number of expired heartbeats |
| `TransactionsSinceLastCheckpoint` | Total number of transactions since last
checkpoint |
| `TransactionsSinceLastLogRoll` | Total number of transactions since last
edit log roll |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02c01815/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index bcc1e25..f86d41e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -747,6 +747,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8730. Clean up the import statements in ClientProtocol.
(Takanobu Asanuma via wheat9)
+HDFS-6682. Add a metric to expose the timestamp of the oldest
+under-replicated block. (aajisaka)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02c01815/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 7dce2a8..64603d0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -171,6 +171,10 @@ public class BlockManager implements BlockStatsMXBean {
public int getPendingDataNodeMessageCount() {
return pendingDNMessages.count();
}
+ /** Used by metrics. */
+ public long getTimeOfTheOldestBlockToBeReplicated() {
+return neededReplications.getTimeOfTheOldestBlockToBeReplicated();
+ }
/**replicationRecheckInterval is how often namenode checks for new
replication work*/
private final long replicationRecheckInterval;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02c01815/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
index 000416e..d8aec99 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/UnderReplicatedBlocks.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/
[04/50] [abbrv] hadoop git commit: YARN-3956. Fix TestNodeManagerHardwareUtils fails on Mac (Varun Vasudev via wangda)
YARN-3956. Fix TestNodeManagerHardwareUtils fails on Mac (Varun Vasudev via
wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06e5dd2c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06e5dd2c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06e5dd2c
Branch: refs/heads/YARN-1197
Commit: 06e5dd2c84c49460884757b56980b1b9c58af996
Parents: 8376ea3
Author: Wangda Tan
Authored: Wed Jul 22 11:59:31 2015 -0700
Committer: Wangda Tan
Committed: Wed Jul 22 12:01:41 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 2 ++
.../server/nodemanager/util/TestNodeManagerHardwareUtils.java | 5 +
2 files changed, 7 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5dd2c/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index eb52745..a5fd4e7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -659,6 +659,8 @@ Release 2.8.0 - UNRELEASED
YARN-3954. Fix
TestYarnConfigurationFields#testCompareConfigurationClassAgainstXml.
(varun saxena via rohithsharmaks)
+YARN-3956. Fix TestNodeManagerHardwareUtils fails on Mac (Varun Vasudev
via wangda)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06e5dd2c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
index 5bf8cb7..84a045d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestNodeManagerHardwareUtils.java
@@ -30,6 +30,11 @@ import org.mockito.Mockito;
public class TestNodeManagerHardwareUtils {
static class TestResourceCalculatorPlugin extends ResourceCalculatorPlugin {
+
+TestResourceCalculatorPlugin() {
+ super(null);
+}
+
@Override
public long getVirtualMemorySize() {
return 0;
[Hadoop Wiki] Update of "BindException" by SteveLoughran
Dear Wiki user,
You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change
notification.
The "BindException" page has been changed by SteveLoughran:
https://wiki.apache.org/hadoop/BindException?action=diff&rev1=7&rev2=8
Comment:
+add the caveat that it's a local config problem
As you cannot have more than one process listening on a TCP port, whatever is
listening is stopping the service coming up. You will need to track down and
stop that process, or change the service you are trying to start up to listen
to a different port.
How to track down the problem
+
1. Identify which host/IP address the program is trying to use.
1. Make sure the hostname is valid:try to ping it; use {{{ifconfig}}} to
list the network interfaces and their IP addresses.
1. Make sure the hostname/IP address is one belonging to the host in
question.
@@ -24, +25 @@
1. As root use {{{netstat -a -t --numeric-ports -p}}} to list the ports that
are in use by number and process. (On OS/X you need to use {{{lsof}}}).
1. Change the configuration of one of the programs to listen on a different
port.
+ Finally, this is not a Hadoop problems, it is a host, network or Hadoop
configuration problem. As it is your cluster, [[YourNetworkYourProblem|only you
can find out and track down the problem.]]. Sorry
+
[Hadoop Wiki] Update of "BindException" by SteveLoughran
Dear Wiki user,
You have subscribed to a wiki page or wiki category on "Hadoop Wiki" for change
notification.
The "BindException" page has been changed by SteveLoughran:
https://wiki.apache.org/hadoop/BindException?action=diff&rev1=7&rev2=8
Comment:
move hostname up for a better workflow
As you cannot have more than one process listening on a TCP port, whatever is
listening is stopping the service coming up. You will need to track down and
stop that process, or change the service you are trying to start up to listen
to a different port.
How to track down the problem
+
1. Identify which host/IP address the program is trying to use.
1. Make sure the hostname is valid:try to ping it; use {{{ifconfig}}} to
list the network interfaces and their IP addresses.
1. Make sure the hostname/IP address is one belonging to the host in
question.
@@ -24, +25 @@
1. As root use {{{netstat -a -t --numeric-ports -p}}} to list the ports that
are in use by number and process. (On OS/X you need to use {{{lsof}}}).
1. Change the configuration of one of the programs to listen on a different
port.
+ Finally, this is not a Hadoop problems, it is a host, network or Hadoop
configuration problem. As it is your cluster, [[YourNetworkYourProblem|only you
can find out and track down the problem.]]. Sorry
+
hadoop git commit: HADOOP-12175. FsShell must load SpanReceierHost to support tracing (Masatake Iwasaki via Colin P. McCabe)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 2ec008eb3 -> 587593566
HADOOP-12175. FsShell must load SpanReceierHost to support tracing (Masatake
Iwasaki via Colin P. McCabe)
(cherry picked from commit 69b095730bd87706ddcbabeea6c513290ee4cdf5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58759356
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58759356
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58759356
Branch: refs/heads/branch-2
Commit: 587593566914a0a4333c9ba87e59fc6488aec122
Parents: 2ec008e
Author: Colin Patrick Mccabe
Authored: Tue Jul 28 12:49:38 2015 -0700
Committer: Colin Patrick Mccabe
Committed: Tue Jul 28 12:56:36 2015 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/fs/FsShell.java | 11 -
.../java/org/apache/hadoop/fs/TestFsShell.java | 24
.../apache/hadoop/tracing/SetSpanReceiver.java | 2 +-
4 files changed, 38 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58759356/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index b7e32ca..e2f79a8 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -526,6 +526,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12245. References to misspelled REMAINING_QUATA in
FileSystemShell.md. (Gabor Liptak via aajisaka)
+HADOOP-12175. FsShell must load SpanReceierHost to support tracing
+(Masatake Iwasaki via Colin P. McCabe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58759356/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index 24b6339..e9c2f73 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.shell.Command;
import org.apache.hadoop.fs.shell.CommandFactory;
import org.apache.hadoop.fs.shell.FsCommand;
+import org.apache.hadoop.tracing.SpanReceiverHost;
import org.apache.hadoop.tools.TableListing;
import org.apache.hadoop.tracing.TraceUtils;
import org.apache.hadoop.util.Tool;
@@ -57,6 +58,9 @@ public class FsShell extends Configured implements Tool {
private final String usagePrefix =
"Usage: hadoop fs [generic options]";
+ private SpanReceiverHost spanReceiverHost;
+ static final String SEHLL_HTRACE_PREFIX = "dfs.shell.htrace.";
+
/**
* Default ctor with no configuration. Be sure to invoke
* {@link #setConf(Configuration)} with a valid configuration prior
@@ -97,6 +101,8 @@ public class FsShell extends Configured implements Tool {
commandFactory.addObject(new Usage(), "-usage");
registerCommands(commandFactory);
}
+this.spanReceiverHost =
+SpanReceiverHost.get(getConf(), SEHLL_HTRACE_PREFIX);
}
protected void registerCommands(CommandFactory factory) {
@@ -279,7 +285,7 @@ public class FsShell extends Configured implements Tool {
// initialize FsShell
init();
traceSampler = new SamplerBuilder(TraceUtils.
-wrapHadoopConf("dfs.shell.htrace.", getConf())).build();
+wrapHadoopConf(SEHLL_HTRACE_PREFIX, getConf())).build();
int exitCode = -1;
if (argv.length < 1) {
printUsage(System.err);
@@ -335,6 +341,9 @@ public class FsShell extends Configured implements Tool {
fs.close();
fs = null;
}
+if (this.spanReceiverHost != null) {
+ this.spanReceiverHost.closeReceivers();
+}
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58759356/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
index 12c8bcf..38e768f 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
@@ -18,6 +18,12 @@
hadoop git commit: HADOOP-12175. FsShell must load SpanReceierHost to support tracing (Masatake Iwasaki via Colin P. McCabe)
Repository: hadoop
Updated Branches:
refs/heads/trunk f17093421 -> 69b095730
HADOOP-12175. FsShell must load SpanReceierHost to support tracing (Masatake
Iwasaki via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69b09573
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69b09573
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69b09573
Branch: refs/heads/trunk
Commit: 69b095730bd87706ddcbabeea6c513290ee4cdf5
Parents: f170934
Author: Colin Patrick Mccabe
Authored: Tue Jul 28 12:49:38 2015 -0700
Committer: Colin Patrick Mccabe
Committed: Tue Jul 28 12:56:15 2015 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/fs/FsShell.java | 11 -
.../java/org/apache/hadoop/fs/TestFsShell.java | 24
.../apache/hadoop/tracing/SetSpanReceiver.java | 2 +-
4 files changed, 38 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b09573/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index aeaa5b9..84e7076 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1020,6 +1020,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12245. References to misspelled REMAINING_QUATA in
FileSystemShell.md. (Gabor Liptak via aajisaka)
+HADOOP-12175. FsShell must load SpanReceierHost to support tracing
+(Masatake Iwasaki via Colin P. McCabe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b09573/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index 24b6339..e9c2f73 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.shell.Command;
import org.apache.hadoop.fs.shell.CommandFactory;
import org.apache.hadoop.fs.shell.FsCommand;
+import org.apache.hadoop.tracing.SpanReceiverHost;
import org.apache.hadoop.tools.TableListing;
import org.apache.hadoop.tracing.TraceUtils;
import org.apache.hadoop.util.Tool;
@@ -57,6 +58,9 @@ public class FsShell extends Configured implements Tool {
private final String usagePrefix =
"Usage: hadoop fs [generic options]";
+ private SpanReceiverHost spanReceiverHost;
+ static final String SEHLL_HTRACE_PREFIX = "dfs.shell.htrace.";
+
/**
* Default ctor with no configuration. Be sure to invoke
* {@link #setConf(Configuration)} with a valid configuration prior
@@ -97,6 +101,8 @@ public class FsShell extends Configured implements Tool {
commandFactory.addObject(new Usage(), "-usage");
registerCommands(commandFactory);
}
+this.spanReceiverHost =
+SpanReceiverHost.get(getConf(), SEHLL_HTRACE_PREFIX);
}
protected void registerCommands(CommandFactory factory) {
@@ -279,7 +285,7 @@ public class FsShell extends Configured implements Tool {
// initialize FsShell
init();
traceSampler = new SamplerBuilder(TraceUtils.
-wrapHadoopConf("dfs.shell.htrace.", getConf())).build();
+wrapHadoopConf(SEHLL_HTRACE_PREFIX, getConf())).build();
int exitCode = -1;
if (argv.length < 1) {
printUsage(System.err);
@@ -335,6 +341,9 @@ public class FsShell extends Configured implements Tool {
fs.close();
fs = null;
}
+if (this.spanReceiverHost != null) {
+ this.spanReceiverHost.closeReceivers();
+}
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69b09573/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
index 12c8bcf..38e768f 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShell.java
@@ -18,6 +18,12 @@
package org.apache.hadoop.fs;
import junit.framework.AssertionFailedE
[3/6] hadoop git commit: HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)
HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/030fcfa9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/030fcfa9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/030fcfa9
Branch: refs/heads/HDFS-7240
Commit: 030fcfa99c345ad57625486eeabedebf2fd4411f
Parents: e21dde5
Author: Arun Suresh
Authored: Mon Jul 27 23:02:03 2015 -0700
Committer: Arun Suresh
Committed: Mon Jul 27 23:02:03 2015 -0700
--
.../apache/hadoop/io/retry/MultiException.java | 49 +++
.../hadoop/io/retry/RetryInvocationHandler.java | 99 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../ha/ConfiguredFailoverProxyProvider.java | 52 ++-
.../ha/RequestHedgingProxyProvider.java | 186 ++
.../markdown/HDFSHighAvailabilityWithNFS.md | 9 +-
.../markdown/HDFSHighAvailabilityWithQJM.md | 10 +-
.../ha/TestRequestHedgingProxyProvider.java | 350 +++
8 files changed, 724 insertions(+), 33 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/030fcfa9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
new file mode 100644
index 000..4963a2d
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.io.retry;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Holder class that clients can use to return multiple exceptions.
+ */
+public class MultiException extends IOException {
+
+ private final Map exes;
+
+ public MultiException(Map exes) {
+this.exes = exes;
+ }
+
+ public Map getExceptions() {
+return exes;
+ }
+
+ @Override
+ public String toString() {
+StringBuilder sb = new StringBuilder("{");
+for (Exception e : exes.values()) {
+ sb.append(e.toString()).append(", ");
+}
+sb.append("}");
+return "MultiException[" + sb.toString() + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/030fcfa9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
index 543567e..9256356 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
@@ -23,6 +23,8 @@ import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -101,7 +103,7 @@ public class RetryInvocationHandler implements
RpcInvocationHandler {
Object ret = invokeMethod(method, args);
hasMadeASuccessfulCall = true;
return ret;
- } catch (Exception e) {
+ } catch (Exception ex) {
boolean isIdempotentOrAtMostOnce = proxyProvider.getInterface()
.getMethod(method.getName(), method.getParameterTypes())
.isAnnotationPresent(Idempotent.class);
@@ -110,15 +112,16 @@ public class RetryInvocationHandler implements
RpcInvocationHandler {
.getMethod(method.getName(), method.getParameterTypes())
.isA
[5/6] hadoop git commit: Merge branch 'trunk' into HDFS-7240
Merge branch 'trunk' into HDFS-7240 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/188d283a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/188d283a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/188d283a Branch: refs/heads/HDFS-7240 Commit: 188d283a4debe2151ee3327ac23bad86b9b6aba3 Parents: 2ebe8c7 f170934 Author: Anu Engineer Authored: Tue Jul 28 11:12:08 2015 -0700 Committer: Anu Engineer Committed: Tue Jul 28 11:12:08 2015 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/io/retry/MultiException.java | 49 +++ .../hadoop/io/retry/RetryInvocationHandler.java | 99 +- .../src/site/markdown/FileSystemShell.md| 2 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../ha/ConfiguredFailoverProxyProvider.java | 52 ++- .../ha/RequestHedgingProxyProvider.java | 186 ++ .../markdown/HDFSHighAvailabilityWithNFS.md | 9 +- .../markdown/HDFSHighAvailabilityWithQJM.md | 10 +- .../ha/TestRequestHedgingProxyProvider.java | 350 +++ hadoop-yarn-project/CHANGES.txt | 6 + .../container-executor/impl/configuration.c | 4 +- .../test/test-container-executor.c | 22 +- .../webapp/CapacitySchedulerPage.java | 5 +- 14 files changed, 755 insertions(+), 44 deletions(-) --
[6/6] hadoop git commit: HDFS-8695. OzoneHandler : Add Bucket REST Interface. (aengineer)
HDFS-8695. OzoneHandler : Add Bucket REST Interface. (aengineer)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7851874
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7851874
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7851874
Branch: refs/heads/HDFS-7240
Commit: c78518749e431684cfc7685e1cf3299751b130e6
Parents: 188d283
Author: Anu Engineer
Authored: Tue Jul 28 11:13:13 2015 -0700
Committer: Anu Engineer
Committed: Tue Jul 28 11:13:13 2015 -0700
--
.../hadoop/ozone/web/exceptions/ErrorTable.java | 5 +
.../hadoop/ozone/web/handlers/BucketArgs.java | 10 -
.../ozone/web/handlers/BucketHandler.java | 193 +
.../web/handlers/BucketProcessTemplate.java | 278 +++
.../apache/hadoop/ozone/web/headers/Header.java | 14 +-
.../hadoop/ozone/web/interfaces/Bucket.java | 133 +
.../ozone/web/interfaces/StorageHandler.java| 85 ++
.../web/localstorage/LocalStorageHandler.java | 109
8 files changed, 815 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7851874/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
index a51dac5..7e75cf0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
@@ -58,6 +58,7 @@ public final class ErrorTable {
new OzoneException(HTTP_BAD_REQUEST, "malformedACL",
"Invalid ACL specified.");
+
public static final OzoneException INVALID_VOLUME_NAME =
new OzoneException(HTTP_BAD_REQUEST, "invalidVolumeName",
"Invalid volume name.");
@@ -81,6 +82,10 @@ public final class ErrorTable {
new OzoneException(HTTP_BAD_REQUEST, "malformedBucketVersion",
"Malformed bucket version or version not unique.");
+ public static final OzoneException MALFORMED_STORAGE_TYPE =
+ new OzoneException(HTTP_BAD_REQUEST, "malformedStorageType",
+ "Invalid storage Type specified.");
+
public static final OzoneException MALFORMED_STORAGE_CLASS =
new OzoneException(HTTP_BAD_REQUEST, "malformedStorageClass",
"Invalid storage class specified.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7851874/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
index 315ae3f..d62c72d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
@@ -164,16 +164,6 @@ public class BucketArgs extends VolumeArgs {
return versioning;
}
- /**
- * Converts a valid String to Enum for ease of use.
- *
- * @param version version string.
- */
- public void setVersioning(String version) {
-if (version != null) {
- this.versioning = OzoneConsts.Versioning.valueOf(version.toUpperCase());
-}
- }
/**
* SetVersioning Info.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7851874/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
new file mode 100644
index 000..2005367
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
@@ -0,0 +1,193 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except
[1/6] hadoop git commit: YARN-3846. RM Web UI queue filter is not working for sub queue. Contributed by Mohammad Shahid Khan
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 2ebe8c7cb -> c78518749
YARN-3846. RM Web UI queue filter is not working for sub queue. Contributed by
Mohammad Shahid Khan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3572ebd7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3572ebd7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3572ebd7
Branch: refs/heads/HDFS-7240
Commit: 3572ebd738aa5fa8b0906d75fb12cc6cbb991573
Parents: 3e6fce9
Author: Jian He
Authored: Mon Jul 27 16:57:11 2015 -0700
Committer: Jian He
Committed: Mon Jul 27 17:12:05 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/resourcemanager/webapp/CapacitySchedulerPage.java| 5 -
2 files changed, 7 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3572ebd7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 534c55a..4f8484a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -695,6 +695,9 @@ Release 2.8.0 - UNRELEASED
YARN-3958. TestYarnConfigurationFields should be moved to hadoop-yarn-api
module. (Varun Saxena via aajisaka)
+YARN-3846. RM Web UI queue filter is not working for sub queue.
+(Mohammad Shahid Khan via jianhe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3572ebd7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 12a3013..d8971b7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -516,7 +516,10 @@ class CapacitySchedulerPage extends RmView {
" $('#cs').bind('select_node.jstree', function(e, data) {",
"var q = $('.q', data.rslt.obj).first().text();",
"if (q == 'Queue: root') q = '';",
- "else q = '^' + q.substr(q.lastIndexOf(':') + 2) + '$';",
+ "else {",
+ " q = q.substr(q.lastIndexOf(':') + 2);",
+ " q = '^' + q.substr(q.lastIndexOf('.') + 1) + '$';",
+ "}",
"$('#apps').dataTable().fnFilter(q, 4, true);",
" });",
" $('#cs').show();",
[2/6] hadoop git commit: HADOOP-12245. References to misspelled REMAINING_QUATA in FileSystemShell.md. Contributed by Gabor Liptak.
HADOOP-12245. References to misspelled REMAINING_QUATA in FileSystemShell.md. Contributed by Gabor Liptak. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e21dde50 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e21dde50 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e21dde50 Branch: refs/heads/HDFS-7240 Commit: e21dde501aa9323b7f34b4bc4ba9d282ec4f2707 Parents: 3572ebd Author: Akira Ajisaka Authored: Tue Jul 28 11:33:10 2015 +0900 Committer: Akira Ajisaka Committed: Tue Jul 28 11:33:10 2015 +0900 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop-common/src/site/markdown/FileSystemShell.md| 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e21dde50/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index baf39e3..aeaa5b9 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1017,6 +1017,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12239. StorageException complaining " no lease ID" when updating FolderLastModifiedTime in WASB. (Duo Xu via cnauroth) +HADOOP-12245. References to misspelled REMAINING_QUATA in +FileSystemShell.md. (Gabor Liptak via aajisaka) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/e21dde50/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md -- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md index 144cb73..fb89ca1 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md @@ -174,7 +174,7 @@ Usage: `hadoop fs -count [-q] [-h] [-v] ` Count the number of directories, files and bytes under the paths that match the specified file pattern. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME -The output columns with -count -q are: QUOTA, REMAINING\_QUATA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME +The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME The -h option shows sizes in human readable format.
[4/6] hadoop git commit: YARN-3982. container-executor parsing of container-executor.cfg broken in trunk and branch-2. Contributed by Varun Vasudev
YARN-3982. container-executor parsing of container-executor.cfg broken
in trunk and branch-2. Contributed by Varun Vasudev
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f1709342
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f1709342
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f1709342
Branch: refs/heads/HDFS-7240
Commit: f17093421521efcbdc813f6f2b8411e45ecc7863
Parents: 030fcfa
Author: Xuan
Authored: Mon Jul 27 23:45:58 2015 -0700
Committer: Xuan
Committed: Mon Jul 27 23:45:58 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../container-executor/impl/configuration.c | 4 ++--
.../test/test-container-executor.c | 22 +---
3 files changed, 20 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4f8484a..b4666e8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -698,6 +698,9 @@ Release 2.8.0 - UNRELEASED
YARN-3846. RM Web UI queue filter is not working for sub queue.
(Mohammad Shahid Khan via jianhe)
+YARN-3982. container-executor parsing of container-executor.cfg broken in
+trunk and branch-2. (Varun Vasudev via xgong)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index 2825367..373dbfd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -284,11 +284,11 @@ char * get_value(const char* key) {
/**
* Function to return an array of values for a key.
- * Value delimiter is assumed to be a '%'.
+ * Value delimiter is assumed to be a ','.
*/
char ** get_values(const char * key) {
char *value = get_value(key);
- return extract_values(value);
+ return extract_values_delim(value, ",");
}
char ** extract_values_delim(char *value, const char *delim) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 99bcf34..001a37d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -145,7 +145,7 @@ void check_pid_file(const char* pid_file, pid_t mypid) {
}
char myPidBuf[33];
- snprintf(myPidBuf, 33, "%" PRId64, (int64_t)mypid);
+ snprintf(myPidBuf, 33, "%" PRId64, (int64_t)(mypid + 1));
if (strncmp(pidBuf, myPidBuf, strlen(myPidBuf)) != 0) {
printf("FAIL: failed to find matching pid in pid file\n");
printf("FAIL: Expected pid %" PRId64 " : Got %.*s", (int64_t)mypid,
@@ -212,15 +212,15 @@ void test_get_app_log_dir() {
free(logdir);
}
-void test_check_user() {
+void test_check_user(int expectedFailure) {
printf("\nTesting test_check_user\n");
struct passwd *user = check_user(username);
- if (user == NULL) {
+ if (user == NULL && !expectedFailure) {
printf("FAIL: failed check for user %s\n", username);
exit(1);
}
free(user);
- if (check_user("lp") != NULL) {
+ if (check_user("lp") != NULL && !expectedFailure) {
printf("FAIL: failed check for system user lp\n");
exit(1);
}
@@ -228,7 +228,7 @@ void test_check_user() {
printf("FAIL: failed check for system user root\n
Git Push Summary
Repository: hadoop Updated Branches: refs/heads/hdfs-7240 [deleted] be57a1345
[2/2] hadoop git commit: HDFS-8695. OzoneHandler : Add Bucket REST Interface. (aengineer)
HDFS-8695. OzoneHandler : Add Bucket REST Interface. (aengineer)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be57a134
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be57a134
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be57a134
Branch: refs/heads/hdfs-7240
Commit: be57a1345fce613093692b3da0e8615412cc0fdc
Parents: 6b3866d
Author: Anu Engineer
Authored: Tue Jul 28 10:56:11 2015 -0700
Committer: Anu Engineer
Committed: Tue Jul 28 10:56:11 2015 -0700
--
.../hadoop/ozone/web/exceptions/ErrorTable.java | 5 +
.../hadoop/ozone/web/handlers/BucketArgs.java | 10 -
.../ozone/web/handlers/BucketHandler.java | 193 +
.../web/handlers/BucketProcessTemplate.java | 278 +++
.../apache/hadoop/ozone/web/headers/Header.java | 14 +-
.../hadoop/ozone/web/interfaces/Bucket.java | 133 +
.../ozone/web/interfaces/StorageHandler.java| 85 ++
.../web/localstorage/LocalStorageHandler.java | 109
8 files changed, 815 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be57a134/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
index a51dac5..7e75cf0 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/exceptions/ErrorTable.java
@@ -58,6 +58,7 @@ public final class ErrorTable {
new OzoneException(HTTP_BAD_REQUEST, "malformedACL",
"Invalid ACL specified.");
+
public static final OzoneException INVALID_VOLUME_NAME =
new OzoneException(HTTP_BAD_REQUEST, "invalidVolumeName",
"Invalid volume name.");
@@ -81,6 +82,10 @@ public final class ErrorTable {
new OzoneException(HTTP_BAD_REQUEST, "malformedBucketVersion",
"Malformed bucket version or version not unique.");
+ public static final OzoneException MALFORMED_STORAGE_TYPE =
+ new OzoneException(HTTP_BAD_REQUEST, "malformedStorageType",
+ "Invalid storage Type specified.");
+
public static final OzoneException MALFORMED_STORAGE_CLASS =
new OzoneException(HTTP_BAD_REQUEST, "malformedStorageClass",
"Invalid storage class specified.");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be57a134/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
index 315ae3f..d62c72d 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketArgs.java
@@ -164,16 +164,6 @@ public class BucketArgs extends VolumeArgs {
return versioning;
}
- /**
- * Converts a valid String to Enum for ease of use.
- *
- * @param version version string.
- */
- public void setVersioning(String version) {
-if (version != null) {
- this.versioning = OzoneConsts.Versioning.valueOf(version.toUpperCase());
-}
- }
/**
* SetVersioning Info.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be57a134/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
new file mode 100644
index 000..2005367
--- /dev/null
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/ozone/web/handlers/BucketHandler.java
@@ -0,0 +1,193 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except
[1/2] hadoop git commit: Merge branch 'trunk' into hdfs-7240
Repository: hadoop Updated Branches: refs/heads/hdfs-7240 [created] be57a1345 Merge branch 'trunk' into hdfs-7240 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b3866d5 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b3866d5 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b3866d5 Branch: refs/heads/hdfs-7240 Commit: 6b3866d5299ad305fad4cbb576bb54dfea71e813 Parents: 2ebe8c7 f170934 Author: Anu Engineer Authored: Tue Jul 28 10:55:34 2015 -0700 Committer: Anu Engineer Committed: Tue Jul 28 10:55:34 2015 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/io/retry/MultiException.java | 49 +++ .../hadoop/io/retry/RetryInvocationHandler.java | 99 +- .../src/site/markdown/FileSystemShell.md| 2 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../ha/ConfiguredFailoverProxyProvider.java | 52 ++- .../ha/RequestHedgingProxyProvider.java | 186 ++ .../markdown/HDFSHighAvailabilityWithNFS.md | 9 +- .../markdown/HDFSHighAvailabilityWithQJM.md | 10 +- .../ha/TestRequestHedgingProxyProvider.java | 350 +++ hadoop-yarn-project/CHANGES.txt | 6 + .../container-executor/impl/configuration.c | 4 +- .../test/test-container-executor.c | 22 +- .../webapp/CapacitySchedulerPage.java | 5 +- 14 files changed, 755 insertions(+), 44 deletions(-) --
[02/10] hadoop git commit: YARN-3853. Add docker container runtime support to LinuxContainterExecutor. Contributed by Sidharta Seethana.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
new file mode 100644
index 000..f9a890e
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java
@@ -0,0 +1,107 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
+
+import org.apache.hadoop.util.StringUtils;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class DockerRunCommand extends DockerCommand {
+ private static final String RUN_COMMAND = "run";
+ private final String image;
+ private List overrrideCommandWithArgs;
+
+ /** The following are mandatory: */
+ public DockerRunCommand(String containerId, String user, String image) {
+super(RUN_COMMAND);
+super.addCommandArguments("--name=" + containerId, "--user=" + user);
+this.image = image;
+ }
+
+ public DockerRunCommand removeContainerOnExit() {
+super.addCommandArguments("--rm");
+return this;
+ }
+
+ public DockerRunCommand detachOnRun() {
+super.addCommandArguments("-d");
+return this;
+ }
+
+ public DockerRunCommand setContainerWorkDir(String workdir) {
+super.addCommandArguments("--workdir=" + workdir);
+return this;
+ }
+
+ public DockerRunCommand setNetworkType(String type) {
+super.addCommandArguments("--net=" + type);
+return this;
+ }
+
+ public DockerRunCommand addMountLocation(String sourcePath, String
+ destinationPath) {
+super.addCommandArguments("-v", sourcePath + ":" + destinationPath);
+return this;
+ }
+
+ public DockerRunCommand setCGroupParent(String parentPath) {
+super.addCommandArguments("--cgroup-parent=" + parentPath);
+return this;
+ }
+
+ public DockerRunCommand addDevice(String sourceDevice, String
+ destinationDevice) {
+super.addCommandArguments("--device=" + sourceDevice + ":" +
+destinationDevice);
+return this;
+ }
+
+ public DockerRunCommand enableDetach() {
+super.addCommandArguments("--detach=true");
+return this;
+ }
+
+ public DockerRunCommand disableDetach() {
+super.addCommandArguments("--detach=false");
+return this;
+ }
+
+ public DockerRunCommand setOverrideCommandWithArgs(
+ List overrideCommandWithArgs) {
+this.overrrideCommandWithArgs = overrideCommandWithArgs;
+return this;
+ }
+
+ @Override
+ public String getCommandWithArguments() {
+List argList = new ArrayList<>();
+
+argList.add(super.getCommandWithArguments());
+argList.add(image);
+
+if (overrrideCommandWithArgs != null) {
+ argList.addAll(overrrideCommandWithArgs);
+}
+
+return StringUtils.join(" ", argList);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerExecutionException.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/runtime/ContainerExecutionException.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containe
[03/10] hadoop git commit: YARN-3853. Add docker container runtime support to LinuxContainterExecutor. Contributed by Sidharta Seethana.
YARN-3853. Add docker container runtime support to LinuxContainterExecutor. Contributed by Sidharta Seethana. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e6fce91 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e6fce91 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e6fce91 Branch: refs/heads/HADOOP-12111 Commit: 3e6fce91a471b4a5099de109582e7c6417e8a822 Parents: f36835f Author: Varun Vasudev Authored: Mon Jul 27 11:57:40 2015 -0700 Committer: Varun Vasudev Committed: Mon Jul 27 11:57:40 2015 -0700 -- hadoop-yarn-project/CHANGES.txt | 4 + .../server/nodemanager/ContainerExecutor.java | 23 +- .../nodemanager/DefaultContainerExecutor.java | 2 +- .../nodemanager/DockerContainerExecutor.java| 2 +- .../nodemanager/LinuxContainerExecutor.java | 222 +++ .../launcher/ContainerLaunch.java | 15 + .../linux/privileged/PrivilegedOperation.java | 46 +++- .../PrivilegedOperationException.java | 30 +- .../privileged/PrivilegedOperationExecutor.java | 30 +- .../linux/resources/CGroupsHandler.java | 8 + .../linux/resources/CGroupsHandlerImpl.java | 12 +- .../runtime/DefaultLinuxContainerRuntime.java | 148 ++ .../DelegatingLinuxContainerRuntime.java| 110 .../runtime/DockerLinuxContainerRuntime.java| 273 +++ .../linux/runtime/LinuxContainerRuntime.java| 38 +++ .../runtime/LinuxContainerRuntimeConstants.java | 69 + .../linux/runtime/docker/DockerClient.java | 82 ++ .../linux/runtime/docker/DockerCommand.java | 66 + .../linux/runtime/docker/DockerLoadCommand.java | 30 ++ .../linux/runtime/docker/DockerRunCommand.java | 107 .../runtime/ContainerExecutionException.java| 85 ++ .../runtime/ContainerRuntime.java | 50 .../runtime/ContainerRuntimeConstants.java | 33 +++ .../runtime/ContainerRuntimeContext.java| 105 +++ .../executor/ContainerLivenessContext.java | 13 + .../executor/ContainerReacquisitionContext.java | 13 + .../executor/ContainerSignalContext.java| 13 + .../executor/ContainerStartContext.java | 23 +- .../TestLinuxContainerExecutorWithMocks.java| 118 +--- .../TestPrivilegedOperationExecutor.java| 8 +- .../runtime/TestDockerContainerRuntime.java | 219 +++ 31 files changed, 1815 insertions(+), 182 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4e54aea..534c55a 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -153,6 +153,10 @@ Release 2.8.0 - UNRELEASED YARN-3852. Add docker container support to container-executor (Abin Shahab via vvasudev) +YARN-3853. Add docker container runtime support to LinuxContainterExecutor. +(Sidharta Seethana via vvasudev) + + IMPROVEMENTS YARN-644. Basic null check is not performed on passed in arguments before http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e6fce91/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 79f9b0d..68bfbbf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -24,8 +24,10 @@ import java.io.OutputStream; import java.io.PrintStream; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.locks.ReentrantReadWriteLock; @@ -39,6 +41,7 @@ import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.a
[04/10] hadoop git commit: YARN-3846. RM Web UI queue filter is not working for sub queue. Contributed by Mohammad Shahid Khan
YARN-3846. RM Web UI queue filter is not working for sub queue. Contributed by
Mohammad Shahid Khan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3572ebd7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3572ebd7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3572ebd7
Branch: refs/heads/HADOOP-12111
Commit: 3572ebd738aa5fa8b0906d75fb12cc6cbb991573
Parents: 3e6fce9
Author: Jian He
Authored: Mon Jul 27 16:57:11 2015 -0700
Committer: Jian He
Committed: Mon Jul 27 17:12:05 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/resourcemanager/webapp/CapacitySchedulerPage.java| 5 -
2 files changed, 7 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3572ebd7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 534c55a..4f8484a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -695,6 +695,9 @@ Release 2.8.0 - UNRELEASED
YARN-3958. TestYarnConfigurationFields should be moved to hadoop-yarn-api
module. (Varun Saxena via aajisaka)
+YARN-3846. RM Web UI queue filter is not working for sub queue.
+(Mohammad Shahid Khan via jianhe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3572ebd7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 12a3013..d8971b7 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -516,7 +516,10 @@ class CapacitySchedulerPage extends RmView {
" $('#cs').bind('select_node.jstree', function(e, data) {",
"var q = $('.q', data.rslt.obj).first().text();",
"if (q == 'Queue: root') q = '';",
- "else q = '^' + q.substr(q.lastIndexOf(':') + 2) + '$';",
+ "else {",
+ " q = q.substr(q.lastIndexOf(':') + 2);",
+ " q = '^' + q.substr(q.lastIndexOf('.') + 1) + '$';",
+ "}",
"$('#apps').dataTable().fnFilter(q, 4, true);",
" });",
" $('#cs').show();",
[08/10] hadoop git commit: Merge branch 'trunk' into HADOOP-12111
Merge branch 'trunk' into HADOOP-12111 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b6953a9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b6953a9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b6953a9 Branch: refs/heads/HADOOP-12111 Commit: 0b6953a9405a1fff7360da8a22174987c2ad9936 Parents: 4d4f288 f170934 Author: Allen Wittenauer Authored: Tue Jul 28 10:42:31 2015 -0700 Committer: Allen Wittenauer Committed: Tue Jul 28 10:42:31 2015 -0700 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 + .../apache/hadoop/io/retry/MultiException.java | 49 +++ .../hadoop/io/retry/RetryInvocationHandler.java | 99 - .../src/site/markdown/FileSystemShell.md| 2 +- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 + .../ha/ConfiguredFailoverProxyProvider.java | 52 ++- .../ha/RequestHedgingProxyProvider.java | 186 + .../markdown/HDFSHighAvailabilityWithNFS.md | 9 +- .../markdown/HDFSHighAvailabilityWithQJM.md | 10 +- .../ha/TestRequestHedgingProxyProvider.java | 350 hadoop-yarn-project/CHANGES.txt | 13 + .../server/nodemanager/ContainerExecutor.java | 23 +- .../nodemanager/DefaultContainerExecutor.java | 2 +- .../nodemanager/DockerContainerExecutor.java| 2 +- .../nodemanager/LinuxContainerExecutor.java | 222 +- .../launcher/ContainerLaunch.java | 15 + .../linux/privileged/PrivilegedOperation.java | 46 +- .../PrivilegedOperationException.java | 30 +- .../privileged/PrivilegedOperationExecutor.java | 30 +- .../linux/resources/CGroupsHandler.java | 8 + .../linux/resources/CGroupsHandlerImpl.java | 12 +- .../runtime/DefaultLinuxContainerRuntime.java | 148 +++ .../DelegatingLinuxContainerRuntime.java| 110 + .../runtime/DockerLinuxContainerRuntime.java| 273 .../linux/runtime/LinuxContainerRuntime.java| 38 ++ .../runtime/LinuxContainerRuntimeConstants.java | 69 +++ .../linux/runtime/docker/DockerClient.java | 82 .../linux/runtime/docker/DockerCommand.java | 66 +++ .../linux/runtime/docker/DockerLoadCommand.java | 30 ++ .../linux/runtime/docker/DockerRunCommand.java | 107 + .../runtime/ContainerExecutionException.java| 85 .../runtime/ContainerRuntime.java | 50 +++ .../runtime/ContainerRuntimeConstants.java | 33 ++ .../runtime/ContainerRuntimeContext.java| 105 + .../executor/ContainerLivenessContext.java | 13 + .../executor/ContainerReacquisitionContext.java | 13 + .../executor/ContainerSignalContext.java| 13 + .../executor/ContainerStartContext.java | 23 +- .../container-executor/impl/configuration.c | 21 +- .../container-executor/impl/configuration.h | 2 + .../impl/container-executor.c | 417 --- .../impl/container-executor.h | 25 +- .../main/native/container-executor/impl/main.c | 97 - .../test/test-container-executor.c | 22 +- .../TestLinuxContainerExecutorWithMocks.java| 118 -- .../TestPrivilegedOperationExecutor.java| 8 +- .../runtime/TestDockerContainerRuntime.java | 219 ++ .../webapp/CapacitySchedulerPage.java | 5 +- 48 files changed, 3050 insertions(+), 307 deletions(-) --
[10/10] hadoop git commit: HADOOP-12254. test-patch.sh should run findbugs if only findbugs-exclude.xml has changed (Kengo Seki via aw)
HADOOP-12254. test-patch.sh should run findbugs if only findbugs-exclude.xml
has changed (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/03335bb4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/03335bb4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/03335bb4
Branch: refs/heads/HADOOP-12111
Commit: 03335bb4d5a047569519ee6775e6edd0c939cf03
Parents: 7c92f0f
Author: Allen Wittenauer
Authored: Tue Jul 28 10:44:37 2015 -0700
Committer: Allen Wittenauer
Committed: Tue Jul 28 10:44:37 2015 -0700
--
dev-support/test-patch.d/findbugs.sh | 19 ++-
1 file changed, 10 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/03335bb4/dev-support/test-patch.d/findbugs.sh
--
diff --git a/dev-support/test-patch.d/findbugs.sh
b/dev-support/test-patch.d/findbugs.sh
index 1d7118b..4fa5428 100755
--- a/dev-support/test-patch.d/findbugs.sh
+++ b/dev-support/test-patch.d/findbugs.sh
@@ -20,13 +20,14 @@ FINDBUGS_WARNINGS_FAIL_PRECHECK=false
add_plugin findbugs
-function findbugs_file_filter
+function findbugs_filefilter
{
local filename=$1
if [[ ${BUILDTOOL} == maven
|| ${BUILDTOOL} == ant ]]; then
-if [[ ${filename} =~ \.java$ ]]; then
+if [[ ${filename} =~ \.java$
+ || ${filename} =~ (^|/)findbugs-exclude.xml$ ]]; then
add_test findbugs
fi
fi
@@ -139,7 +140,7 @@ function findbugs_runner
savestop=$(stop_clock)
MODULE_STATUS_TIMER[${i}]=${savestop}
module_status ${i} -1 "" "${name}/${module} cannot run
setBugDatabaseInfo from findbugs"
- ((retval = retval + 1))
+ ((result=result+1))
((i=i+1))
continue
fi
@@ -151,7 +152,7 @@ function findbugs_runner
savestop=$(stop_clock)
MODULE_STATUS_TIMER[${i}]=${savestop}
module_status ${i} -1 "" "${name}/${module} cannot run convertXmlToText
from findbugs"
- ((result = result + 1))
+ ((result=result+1))
fi
if [[ -z ${FINDBUGS_VERSION}
@@ -182,7 +183,7 @@ function findbugs_preapply
local i=0
local warnings_file
local module_findbugs_warnings
- local results=0
+ local result=0
big_console_header "Pre-patch findbugs detection"
@@ -199,7 +200,7 @@ function findbugs_preapply
fi
findbugs_runner branch
- results=$?
+ result=$?
if [[ "${FINDBUGS_WARNINGS_FAIL_PRECHECK}" == "true" ]]; then
until [[ $i -eq ${#MODULE[@]} ]]; do
@@ -222,7 +223,7 @@ function findbugs_preapply
if [[ ${module_findbugs_warnings} -gt 0 ]] ; then
module_status ${i} -1 "branch-findbugs-${fn}.html" "${module} in
${PATCH_BRANCH} cannot run convertXmlToText from findbugs"
-((results=results+1))
+((result=result+1))
fi
savestop=$(stop_clock)
MODULE_STATUS_TIMER[${i}]=${savestop}
@@ -231,7 +232,7 @@ function findbugs_preapply
modules_messages branch findbugs true
fi
- if [[ ${results} != 0 ]]; then
+ if [[ ${result} != 0 ]]; then
return 1
fi
return 0
@@ -256,7 +257,7 @@ function findbugs_postinstall
local firstpart
local secondpart
local i=0
- local results=0
+ local result=0
local savestop
big_console_header "Patch findbugs detection"
[09/10] hadoop git commit: HADOOP-12273. releasedocmaker.py fails with stacktrace if --project option is not specified (Kengo Seki via aw)
HADOOP-12273. releasedocmaker.py fails with stacktrace if --project option is
not specified (Kengo Seki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c92f0fe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c92f0fe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c92f0fe
Branch: refs/heads/HADOOP-12111
Commit: 7c92f0fe9176c55f7c16bc20211c854cdbfe0141
Parents: 0b6953a
Author: Allen Wittenauer
Authored: Tue Jul 28 10:43:22 2015 -0700
Committer: Allen Wittenauer
Committed: Tue Jul 28 10:43:22 2015 -0700
--
dev-support/releasedocmaker.py | 8 ++--
1 file changed, 2 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c92f0fe/dev-support/releasedocmaker.py
--
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index d2e5dda..c59ae99 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -400,12 +400,6 @@ def main():
(options, args) = parser.parse_args()
if (options.versions is None):
-options.versions = []
-
- if (len(args) > 2):
-options.versions.append(args[2])
-
- if (len(options.versions) <= 0):
parser.error("At least one version needs to be supplied")
proxy = urllib2.ProxyHandler()
@@ -413,6 +407,8 @@ def main():
urllib2.install_opener(opener)
projects = options.projects
+ if projects is None:
+parser.error("At least one project needs to be supplied")
if (options.range is True):
versions = [ Version(v) for v in GetVersions(options.versions,
projects).getlist() ]
[01/10] hadoop git commit: YARN-3852. Add docker container support to container-executor. Contributed by Abin Shahab.
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-12111 4d4f288d3 -> 03335bb4d
YARN-3852. Add docker container support to container-executor. Contributed by
Abin Shahab.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f36835ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f36835ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f36835ff
Branch: refs/heads/HADOOP-12111
Commit: f36835ff9b878fa20fe58a30f9d1e8c47702d6d2
Parents: 2196e39
Author: Varun Vasudev
Authored: Mon Jul 27 10:12:30 2015 -0700
Committer: Varun Vasudev
Committed: Mon Jul 27 10:14:51 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../container-executor/impl/configuration.c | 17 +-
.../container-executor/impl/configuration.h | 2 +
.../impl/container-executor.c | 417 ---
.../impl/container-executor.h | 25 +-
.../main/native/container-executor/impl/main.c | 97 -
6 files changed, 480 insertions(+), 81 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f36835ff/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3b7d8a8..4e54aea 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -150,6 +150,9 @@ Release 2.8.0 - UNRELEASED
YARN-3656. LowCost: A Cost-Based Placement Agent for YARN Reservations.
(Jonathan Yaniv and Ishai Menache via curino)
+YARN-3852. Add docker container support to container-executor
+(Abin Shahab via vvasudev)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f36835ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index eaa1f19..2825367 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -291,27 +291,23 @@ char ** get_values(const char * key) {
return extract_values(value);
}
-/**
- * Extracts array of values from the '%' separated list of values.
- */
-char ** extract_values(char *value) {
+char ** extract_values_delim(char *value, const char *delim) {
char ** toPass = NULL;
char *tempTok = NULL;
char *tempstr = NULL;
int size = 0;
int toPassSize = MAX_SIZE;
-
//first allocate any array of 10
if(value != NULL) {
toPass = (char **) malloc(sizeof(char *) * toPassSize);
-tempTok = strtok_r((char *)value, "%", &tempstr);
+tempTok = strtok_r((char *)value, delim, &tempstr);
while (tempTok != NULL) {
toPass[size++] = tempTok;
if(size == toPassSize) {
toPassSize += MAX_SIZE;
toPass = (char **) realloc(toPass,(sizeof(char *) * toPassSize));
}
- tempTok = strtok_r(NULL, "%", &tempstr);
+ tempTok = strtok_r(NULL, delim, &tempstr);
}
}
if (toPass != NULL) {
@@ -320,6 +316,13 @@ char ** extract_values(char *value) {
return toPass;
}
+/**
+ * Extracts array of values from the '%' separated list of values.
+ */
+char ** extract_values(char *value) {
+ extract_values_delim(value, "%");
+}
+
// free an entry set of values
void free_values(char** values) {
if (*values != NULL) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f36835ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
index 133e67b..390a5b5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.h
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/m
[06/10] hadoop git commit: HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)
HDFS-7858. Improve HA Namenode Failover detection on the client. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/030fcfa9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/030fcfa9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/030fcfa9
Branch: refs/heads/HADOOP-12111
Commit: 030fcfa99c345ad57625486eeabedebf2fd4411f
Parents: e21dde5
Author: Arun Suresh
Authored: Mon Jul 27 23:02:03 2015 -0700
Committer: Arun Suresh
Committed: Mon Jul 27 23:02:03 2015 -0700
--
.../apache/hadoop/io/retry/MultiException.java | 49 +++
.../hadoop/io/retry/RetryInvocationHandler.java | 99 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../ha/ConfiguredFailoverProxyProvider.java | 52 ++-
.../ha/RequestHedgingProxyProvider.java | 186 ++
.../markdown/HDFSHighAvailabilityWithNFS.md | 9 +-
.../markdown/HDFSHighAvailabilityWithQJM.md | 10 +-
.../ha/TestRequestHedgingProxyProvider.java | 350 +++
8 files changed, 724 insertions(+), 33 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/030fcfa9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
new file mode 100644
index 000..4963a2d
--- /dev/null
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/MultiException.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.hadoop.io.retry;
+
+import java.io.IOException;
+import java.util.Map;
+
+/**
+ * Holder class that clients can use to return multiple exceptions.
+ */
+public class MultiException extends IOException {
+
+ private final Map exes;
+
+ public MultiException(Map exes) {
+this.exes = exes;
+ }
+
+ public Map getExceptions() {
+return exes;
+ }
+
+ @Override
+ public String toString() {
+StringBuilder sb = new StringBuilder("{");
+for (Exception e : exes.values()) {
+ sb.append(e.toString()).append(", ");
+}
+sb.append("}");
+return "MultiException[" + sb.toString() + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/030fcfa9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
index 543567e..9256356 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/retry/RetryInvocationHandler.java
@@ -23,6 +23,8 @@ import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -101,7 +103,7 @@ public class RetryInvocationHandler implements
RpcInvocationHandler {
Object ret = invokeMethod(method, args);
hasMadeASuccessfulCall = true;
return ret;
- } catch (Exception e) {
+ } catch (Exception ex) {
boolean isIdempotentOrAtMostOnce = proxyProvider.getInterface()
.getMethod(method.getName(), method.getParameterTypes())
.isAnnotationPresent(Idempotent.class);
@@ -110,15 +112,16 @@ public class RetryInvocationHandler implements
RpcInvocationHandler {
.getMethod(method.getName(), method.getParameterTypes())
.
[05/10] hadoop git commit: HADOOP-12245. References to misspelled REMAINING_QUATA in FileSystemShell.md. Contributed by Gabor Liptak.
HADOOP-12245. References to misspelled REMAINING_QUATA in FileSystemShell.md. Contributed by Gabor Liptak. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e21dde50 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e21dde50 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e21dde50 Branch: refs/heads/HADOOP-12111 Commit: e21dde501aa9323b7f34b4bc4ba9d282ec4f2707 Parents: 3572ebd Author: Akira Ajisaka Authored: Tue Jul 28 11:33:10 2015 +0900 Committer: Akira Ajisaka Committed: Tue Jul 28 11:33:10 2015 +0900 -- hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++ .../hadoop-common/src/site/markdown/FileSystemShell.md| 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/e21dde50/hadoop-common-project/hadoop-common/CHANGES.txt -- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index baf39e3..aeaa5b9 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -1017,6 +1017,9 @@ Release 2.8.0 - UNRELEASED HADOOP-12239. StorageException complaining " no lease ID" when updating FolderLastModifiedTime in WASB. (Duo Xu via cnauroth) +HADOOP-12245. References to misspelled REMAINING_QUATA in +FileSystemShell.md. (Gabor Liptak via aajisaka) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/e21dde50/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md -- diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md index 144cb73..fb89ca1 100644 --- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md +++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md @@ -174,7 +174,7 @@ Usage: `hadoop fs -count [-q] [-h] [-v] ` Count the number of directories, files and bytes under the paths that match the specified file pattern. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME -The output columns with -count -q are: QUOTA, REMAINING\_QUATA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME +The output columns with -count -q are: QUOTA, REMAINING\_QUOTA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME The -h option shows sizes in human readable format.
[07/10] hadoop git commit: YARN-3982. container-executor parsing of container-executor.cfg broken in trunk and branch-2. Contributed by Varun Vasudev
YARN-3982. container-executor parsing of container-executor.cfg broken
in trunk and branch-2. Contributed by Varun Vasudev
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f1709342
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f1709342
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f1709342
Branch: refs/heads/HADOOP-12111
Commit: f17093421521efcbdc813f6f2b8411e45ecc7863
Parents: 030fcfa
Author: Xuan
Authored: Mon Jul 27 23:45:58 2015 -0700
Committer: Xuan
Committed: Mon Jul 27 23:45:58 2015 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../container-executor/impl/configuration.c | 4 ++--
.../test/test-container-executor.c | 22 +---
3 files changed, 20 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4f8484a..b4666e8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -698,6 +698,9 @@ Release 2.8.0 - UNRELEASED
YARN-3846. RM Web UI queue filter is not working for sub queue.
(Mohammad Shahid Khan via jianhe)
+YARN-3982. container-executor parsing of container-executor.cfg broken in
+trunk and branch-2. (Varun Vasudev via xgong)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
index 2825367..373dbfd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/configuration.c
@@ -284,11 +284,11 @@ char * get_value(const char* key) {
/**
* Function to return an array of values for a key.
- * Value delimiter is assumed to be a '%'.
+ * Value delimiter is assumed to be a ','.
*/
char ** get_values(const char * key) {
char *value = get_value(key);
- return extract_values(value);
+ return extract_values_delim(value, ",");
}
char ** extract_values_delim(char *value, const char *delim) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1709342/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 99bcf34..001a37d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -145,7 +145,7 @@ void check_pid_file(const char* pid_file, pid_t mypid) {
}
char myPidBuf[33];
- snprintf(myPidBuf, 33, "%" PRId64, (int64_t)mypid);
+ snprintf(myPidBuf, 33, "%" PRId64, (int64_t)(mypid + 1));
if (strncmp(pidBuf, myPidBuf, strlen(myPidBuf)) != 0) {
printf("FAIL: failed to find matching pid in pid file\n");
printf("FAIL: Expected pid %" PRId64 " : Got %.*s", (int64_t)mypid,
@@ -212,15 +212,15 @@ void test_get_app_log_dir() {
free(logdir);
}
-void test_check_user() {
+void test_check_user(int expectedFailure) {
printf("\nTesting test_check_user\n");
struct passwd *user = check_user(username);
- if (user == NULL) {
+ if (user == NULL && !expectedFailure) {
printf("FAIL: failed check for user %s\n", username);
exit(1);
}
free(user);
- if (check_user("lp") != NULL) {
+ if (check_user("lp") != NULL && !expectedFailure) {
printf("FAIL: failed check for system user lp\n");
exit(1);
}
@@ -228,7 +228,7 @@ void test_check_user() {
printf("FAIL: failed check for system user roo
