[1/2] hadoop git commit: HADOOP-11446 S3AOutputStream should use shared thread pool to avoid OutOfMemoryError
Repository: hadoop
Updated Branches:
refs/heads/branch-2 5782f9e39 - 2800e73c2
refs/heads/trunk 21c6f0183 - 27d839586
HADOOP-11446 S3AOutputStream should use shared thread pool to avoid
OutOfMemoryError
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2800e73c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2800e73c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2800e73c
Branch: refs/heads/branch-2
Commit: 2800e73c2789f3d41fab618b35a25c59a45bf947
Parents: 5782f9e
Author: Steve Loughran ste...@apache.org
Authored: Mon Jan 5 12:59:48 2015 +
Committer: Steve Loughran ste...@apache.org
Committed: Mon Jan 5 12:59:48 2015 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/fs/s3a/Constants.java | 17
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 91 ++--
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 13 +--
4 files changed, 109 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2800e73c/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 635b0b0..451ce61 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -313,6 +313,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11039. ByteBufferReadable API doc is inconsistent with the
implementations. (Yi Liu via Colin P. McCabe)
+HADOOP-11446. S3AOutputStream should use shared thread pool to
+avoid OutOfMemoryError. (Ted Yu via stevel)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2800e73c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index ee4bf68..f1b5d3d 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -41,6 +41,23 @@ public class Constants {
public static final String MAX_PAGING_KEYS = fs.s3a.paging.maximum;
public static final int DEFAULT_MAX_PAGING_KEYS = 5000;
+ // the maximum number of threads to allow in the pool used by TransferManager
+ public static final String MAX_THREADS = fs.s3a.threads.max;
+ public static final int DEFAULT_MAX_THREADS = 256;
+
+ // the number of threads to keep in the pool used by TransferManager
+ public static final String CORE_THREADS = fs.s3a.threads.core;
+ public static final int DEFAULT_CORE_THREADS = DEFAULT_MAXIMUM_CONNECTIONS;
+
+ // when the number of threads is greater than the core, this is the maximum
time
+ // that excess idle threads will wait for new tasks before terminating.
+ public static final String KEEPALIVE_TIME = fs.s3a.threads.keepalivetime;
+ public static final int DEFAULT_KEEPALIVE_TIME = 60;
+
+ // the maximum number of tasks that the LinkedBlockingQueue can hold
+ public static final String MAX_TOTAL_TASKS = fs.s3a.max.total.tasks;
+ public static final int DEFAULT_MAX_TOTAL_TASKS = 1000;
+
// size of each of or multipart pieces in bytes
public static final String MULTIPART_SIZE = fs.s3a.multipart.size;
public static final long DEFAULT_MULTIPART_SIZE = 104857600; // 100 MB
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2800e73c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 457351d..e6b1557 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -26,6 +26,11 @@ import java.net.URI;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.s3.S3Credentials;
@@ -77,6 +82,7 @@ public class S3AFileSystem extends FileSystem {
private String bucket;
private int maxKeys;
private long partSize;
+ private
[2/2] hadoop git commit: HADOOP-11446 S3AOutputStream should use shared thread pool to avoid OutOfMemoryError
HADOOP-11446 S3AOutputStream should use shared thread pool to avoid
OutOfMemoryError
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27d83958
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27d83958
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27d83958
Branch: refs/heads/trunk
Commit: 27d8395867f665fea1360087325cda5ed70efd0c
Parents: 21c6f01
Author: Steve Loughran ste...@apache.org
Authored: Mon Jan 5 12:59:48 2015 +
Committer: Steve Loughran ste...@apache.org
Committed: Mon Jan 5 13:00:01 2015 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/fs/s3a/Constants.java | 17
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 91 ++--
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 13 +--
4 files changed, 109 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d83958/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index ec75e8d..baf68d6 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -663,6 +663,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11039. ByteBufferReadable API doc is inconsistent with the
implementations. (Yi Liu via Colin P. McCabe)
+HADOOP-11446. S3AOutputStream should use shared thread pool to
+avoid OutOfMemoryError. (Ted Yu via stevel)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d83958/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index ee4bf68..f1b5d3d 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -41,6 +41,23 @@ public class Constants {
public static final String MAX_PAGING_KEYS = fs.s3a.paging.maximum;
public static final int DEFAULT_MAX_PAGING_KEYS = 5000;
+ // the maximum number of threads to allow in the pool used by TransferManager
+ public static final String MAX_THREADS = fs.s3a.threads.max;
+ public static final int DEFAULT_MAX_THREADS = 256;
+
+ // the number of threads to keep in the pool used by TransferManager
+ public static final String CORE_THREADS = fs.s3a.threads.core;
+ public static final int DEFAULT_CORE_THREADS = DEFAULT_MAXIMUM_CONNECTIONS;
+
+ // when the number of threads is greater than the core, this is the maximum
time
+ // that excess idle threads will wait for new tasks before terminating.
+ public static final String KEEPALIVE_TIME = fs.s3a.threads.keepalivetime;
+ public static final int DEFAULT_KEEPALIVE_TIME = 60;
+
+ // the maximum number of tasks that the LinkedBlockingQueue can hold
+ public static final String MAX_TOTAL_TASKS = fs.s3a.max.total.tasks;
+ public static final int DEFAULT_MAX_TOTAL_TASKS = 1000;
+
// size of each of or multipart pieces in bytes
public static final String MULTIPART_SIZE = fs.s3a.multipart.size;
public static final long DEFAULT_MULTIPART_SIZE = 104857600; // 100 MB
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d83958/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 457351d..e6b1557 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -26,6 +26,11 @@ import java.net.URI;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.s3.S3Credentials;
@@ -77,6 +82,7 @@ public class S3AFileSystem extends FileSystem {
private String bucket;
private int maxKeys;
private long partSize;
+ private TransferManager transfers;
private int partSizeThreshold;
public static final Logger LOG =
[2/2] hadoop git commit: HADOOP-11390 Metrics 2 ganglia provider to include hostname in unresolved address problems. (Varun Saxena via stevel)
HADOOP-11390 Metrics 2 ganglia provider to include hostname in unresolved
address problems. (Varun Saxena via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2403634d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2403634d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2403634d
Branch: refs/heads/trunk
Commit: 2403634de02a6948791575b63e7ef1562dbad241
Parents: 27d8395
Author: Steve Loughran ste...@apache.org
Authored: Mon Jan 5 16:09:25 2015 +
Committer: Steve Loughran ste...@apache.org
Committed: Mon Jan 5 16:09:36 2015 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java | 7 +++
2 files changed, 10 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2403634d/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index baf68d6..0c76894 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -455,6 +455,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11455. KMS and Credential CLI should request confirmation for
deletion by default. (Charles Lamb via yliu)
+HADOOP-11390 Metrics 2 ganglia provider to include hostname in
+unresolved address problems. (Varun Saxena via stevel)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2403634d/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
index 164ea08..88ad647 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.metrics2.sink.ganglia;
import java.io.IOException;
import java.net.DatagramPacket;
import java.net.DatagramSocket;
+import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.net.SocketException;
import java.net.UnknownHostException;
@@ -257,6 +258,12 @@ public abstract class AbstractGangliaSink implements
MetricsSink {
protected void emitToGangliaHosts() throws IOException {
try {
for (SocketAddress socketAddress : metricsServers) {
+if (socketAddress == null || !(socketAddress instanceof
InetSocketAddress))
+ throw new IllegalArgumentException(Unsupported Address type);
+InetSocketAddress inetAddress = (InetSocketAddress)socketAddress;
+if(inetAddress.isUnresolved()) {
+ throw new UnknownHostException(Unresolved host: + inetAddress);
+}
DatagramPacket packet =
new DatagramPacket(buffer, offset, socketAddress);
datagramSocket.send(packet);
hadoop git commit: YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the scheduler page. (Ashwin Shankar and Wei Yan via kasha)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c6cf74898 - d070597a0
YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the
scheduler page. (Ashwin Shankar and Wei Yan via kasha)
(cherry picked from commit 270a271f53f52a1f33e63a0520ae6032536dd4c6)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d070597a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d070597a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d070597a
Branch: refs/heads/branch-2
Commit: d070597a0ee9d9566855e5aa2d07aedcc0bcee89
Parents: c6cf748
Author: Karthik Kambatla ka...@apache.org
Authored: Fri Aug 29 17:15:38 2014 -0700
Committer: Karthik Kambatla ka...@apache.org
Committed: Tue Jan 6 03:28:48 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 5 +++
.../webapp/FairSchedulerPage.java | 31 --
.../webapp/dao/FairSchedulerQueueInfo.java | 34
.../src/site/apt/FairScheduler.apt.vm | 20
4 files changed, 67 insertions(+), 23 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d070597a/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 83acc08..c737a9a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -129,6 +129,9 @@ Release 2.7.0 - UNRELEASED
YARN-2998. Abstract out scheduler independent PlanFollower components.
(Anubhav Dhoot via kasha)
+YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the
+scheduler page. (Ashwin Shankar and Wei Yan via kasha)
+
OPTIMIZATIONS
@@ -522,6 +525,7 @@ Release 2.6.0 - 2014-11-18
YARN-2406. Move RM recovery related proto to
yarn_server_resourcemanager_recovery.proto. (Tsuyoshi Ozawa via jianhe)
+ HEAD
YARN-1506. Changed RMNode/SchedulerNode to update resource with event
notification. (Junping Du via jianhe)
@@ -657,6 +661,7 @@ Release 2.6.0 - 2014-11-18
YARN-2635. TestRM, TestRMRestart, TestClientToAMTokens should run
with both CS and FS. (Wei Yan and kasha via kasha)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d070597a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
index 0d3c297..8c54f4e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/FairSchedulerPage.java
@@ -44,10 +44,12 @@ public class FairSchedulerPage extends RmView {
static final float Q_MAX_WIDTH = 0.8f;
static final float Q_STATS_POS = Q_MAX_WIDTH + 0.05f;
static final String Q_END = left:101%;
- static final String Q_GIVEN = left:0%;background:none;border:1px dashed
rgba(0,0,0,0.25);
+ static final String Q_GIVEN = left:0%;background:none;border:1px solid
rgba(0,0,0,1);
+ static final String Q_INSTANTANEOUS_FS = left:0%;background:none;border:1px
dashed rgba(0,0,0,1);
static final String Q_OVER = background:rgba(255, 140, 0, 0.8);
static final String Q_UNDER = background:rgba(50, 205, 50, 0.8);
-
+ static final String STEADY_FAIR_SHARE = Steady Fair Share;
+ static final String INSTANTANEOUS_FAIR_SHARE = Instantaneous Fair Share;
@RequestScoped
static class FSQInfo {
FairSchedulerQueueInfo qinfo;
@@ -73,8 +75,8 @@ public class FairSchedulerPage extends RmView {
if (maxApps Integer.MAX_VALUE) {
ri._(Max Running Applications:, qinfo.getMaxApplications());
}
- ri._(Fair Share:, qinfo.getFairShare().toString());
-
+ ri._(STEADY_FAIR_SHARE + :, qinfo.getSteadyFairShare().toString());
+ ri._(INSTANTANEOUS_FAIR_SHARE + :, qinfo.getFairShare().toString());
html._(InfoBlock.class);
// clear the info contents so this queue's info doesn't accumulate into
another queue's info
@@ -95,16 +97,21 @@ public
hadoop git commit: HADOOP-11402. Negative user-to-group cache entries are never cleared for never-again-accessed users. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/trunk 94d342e60 - 53caeaa16
HADOOP-11402. Negative user-to-group cache entries are never cleared for
never-again-accessed users. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53caeaa1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53caeaa1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53caeaa1
Branch: refs/heads/trunk
Commit: 53caeaa16b1450b54e994c77f5d0c8a767b88d57
Parents: 94d342e
Author: Benoy Antony be...@apache.org
Authored: Mon Jan 5 15:06:46 2015 -0800
Committer: Benoy Antony be...@apache.org
Committed: Mon Jan 5 15:06:46 2015 -0800
--
.../java/org/apache/hadoop/security/Groups.java | 36 +--
.../hadoop/security/TestGroupsCaching.java | 48
2 files changed, 71 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53caeaa1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index f3c5094..9fd39b0 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -23,12 +23,14 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Ticker;
import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Cache;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -60,14 +62,13 @@ public class Groups {
private final GroupMappingServiceProvider impl;
private final LoadingCacheString, ListString cache;
- private final ConcurrentHashMapString, Long negativeCacheMask =
-new ConcurrentHashMapString, Long();
private final MapString, ListString staticUserToGroupsMap =
new HashMapString, ListString();
private final long cacheTimeout;
private final long negativeCacheTimeout;
private final long warningDeltaMs;
private final Timer timer;
+ private SetString negativeCache;
public Groups(Configuration conf) {
this(conf, new Timer());
@@ -99,11 +100,24 @@ public class Groups {
.expireAfterWrite(10 * cacheTimeout, TimeUnit.MILLISECONDS)
.build(new GroupCacheLoader());
+if(negativeCacheTimeout 0) {
+ CacheString, Boolean tempMap = CacheBuilder.newBuilder()
+.expireAfterWrite(negativeCacheTimeout, TimeUnit.MILLISECONDS)
+.ticker(new TimerToTickerAdapter(timer))
+.build();
+ negativeCache = Collections.newSetFromMap(tempMap.asMap());
+}
+
if(LOG.isDebugEnabled())
LOG.debug(Group mapping impl= + impl.getClass().getName() +
; cacheTimeout= + cacheTimeout + ; warningDeltaMs= +
warningDeltaMs);
}
+
+ @VisibleForTesting
+ SetString getNegativeCache() {
+return negativeCache;
+ }
/*
* Parse the hadoop.user.group.static.mapping.overrides configuration to
@@ -159,13 +173,8 @@ public class Groups {
// Check the negative cache first
if (isNegativeCacheEnabled()) {
- Long expirationTime = negativeCacheMask.get(user);
- if (expirationTime != null) {
-if (timer.monotonicNow() expirationTime) {
- throw noGroupsForUser(user);
-} else {
- negativeCacheMask.remove(user, expirationTime);
-}
+ if (negativeCache.contains(user)) {
+throw noGroupsForUser(user);
}
}
@@ -212,8 +221,7 @@ public class Groups {
if (groups.isEmpty()) {
if (isNegativeCacheEnabled()) {
- long expirationTime = timer.monotonicNow() + negativeCacheTimeout;
- negativeCacheMask.put(user, expirationTime);
+ negativeCache.add(user);
}
// We throw here to prevent Cache from retaining an empty group
@@ -252,7 +260,9 @@ public class Groups {
LOG.warn(Error refreshing groups cache, e);
}
cache.invalidateAll();
-negativeCacheMask.clear();
+if(isNegativeCacheEnabled()) {
+ negativeCache.clear();
+}
}
/**
hadoop git commit: HADOOP-10908. Common needs updates for shell rewrite (aw)
Repository: hadoop
Updated Branches:
refs/heads/trunk 41d72cbd4 - 94d342e60
HADOOP-10908. Common needs updates for shell rewrite (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94d342e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94d342e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94d342e6
Branch: refs/heads/trunk
Commit: 94d342e607e1db317bae7af86a34ae7cd3860348
Parents: 41d72cb
Author: Allen Wittenauer a...@apache.org
Authored: Mon Jan 5 14:26:41 2015 -0800
Committer: Allen Wittenauer a...@apache.org
Committed: Mon Jan 5 14:26:41 2015 -0800
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../src/site/apt/ClusterSetup.apt.vm| 348 ---
.../src/site/apt/CommandsManual.apt.vm | 316 +
.../src/site/apt/FileSystemShell.apt.vm | 313 ++---
.../src/site/apt/SingleCluster.apt.vm | 20 +-
5 files changed, 534 insertions(+), 465 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94d342e6/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0c76894..40e8d29 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -344,6 +344,8 @@ Trunk (Unreleased)
HADOOP-11397. Can't override HADOOP_IDENT_STRING (Kengo Seki via aw)
+HADOOP-10908. Common needs updates for shell rewrite (aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94d342e6/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
--
diff --git
a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
index f5f1deb..52b0552 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
@@ -11,83 +11,81 @@
~~ limitations under the License. See accompanying LICENSE file.
---
- Hadoop Map Reduce Next Generation-${project.version} - Cluster Setup
+ Hadoop ${project.version} - Cluster Setup
---
---
${maven.build.timestamp}
%{toc|section=1|fromDepth=0}
-Hadoop MapReduce Next Generation - Cluster Setup
+Hadoop Cluster Setup
* {Purpose}
- This document describes how to install, configure and manage non-trivial
+ This document describes how to install and configure
Hadoop clusters ranging from a few nodes to extremely large clusters
- with thousands of nodes.
+ with thousands of nodes. To play with Hadoop, you may first want to
+ install it on a single machine (see {{{./SingleCluster.html}Single Node
Setup}}).
- To play with Hadoop, you may first want to install it on a single
- machine (see {{{./SingleCluster.html}Single Node Setup}}).
+ This document does not cover advanced topics such as
{{{./SecureMode.html}Security}} or
+ High Availability.
* {Prerequisites}
- Download a stable version of Hadoop from Apache mirrors.
+ * Install Java. See the
{{{http://wiki.apache.org/hadoop/HadoopJavaVersions}Hadoop Wiki}} for known
good versions.
+ * Download a stable version of Hadoop from Apache mirrors.
* {Installation}
Installing a Hadoop cluster typically involves unpacking the software on all
- the machines in the cluster or installing RPMs.
+ the machines in the cluster or installing it via a packaging system as
+ appropriate for your operating system. It is important to divide up the
hardware
+ into functions.
Typically one machine in the cluster is designated as the NameNode and
- another machine the as ResourceManager, exclusively. These are the masters.
+ another machine the as ResourceManager, exclusively. These are the masters.
Other
+ services (such as Web App Proxy Server and MapReduce Job History server) are
usually
+ run either on dedicated hardware or on shared infrastrucutre, depending upon
the load.
The rest of the machines in the cluster act as both DataNode and NodeManager.
These are the slaves.
-* {Running Hadoop in Non-Secure Mode}
+* {Configuring Hadoop in Non-Secure Mode}
- The following sections describe how to configure a Hadoop cluster.
-
- {Configuration Files}
-
-Hadoop configuration is driven by two types of important configuration
files:
+Hadoop's Java configuration is driven by two types of important
configuration files:
* Read-only default configuration - core-default.xml,
hadoop git commit: YARN-2881. [YARN-2574] Implement PlanFollower for FairScheduler. (Anubhav Dhoot via kasha)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 0c2d996c2 - 7adffad2b
YARN-2881. [YARN-2574] Implement PlanFollower for FairScheduler. (Anubhav Dhoot
via kasha)
(cherry picked from commit 0c4b11267717eb451fa6ed4c586317f2db32fbd5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7adffad2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7adffad2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7adffad2
Branch: refs/heads/branch-2
Commit: 7adffad2bb17338c266fe1d3320f59b0c89fce4a
Parents: 0c2d996
Author: Karthik Kambatla ka...@apache.org
Authored: Tue Jan 6 04:41:45 2015 +0530
Committer: Karthik Kambatla ka...@apache.org
Committed: Tue Jan 6 04:42:55 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 2 +
.../reservation/AbstractReservationSystem.java | 2 +
.../AbstractSchedulerPlanFollower.java | 3 +-
.../reservation/FairSchedulerPlanFollower.java | 141 +
.../reservation/ReservationConstants.java | 28 +++
.../scheduler/capacity/CapacityScheduler.java | 6 +-
.../scheduler/capacity/PlanQueue.java | 2 -
.../scheduler/fair/AllocationConfiguration.java | 14 ++
.../scheduler/fair/FSLeafQueue.java | 10 +
.../scheduler/fair/FairScheduler.java | 104 +-
.../scheduler/fair/QueueManager.java| 13 +-
.../fair/ReservationQueueConfiguration.java | 11 +
.../reservation/ReservationSystemTestUtil.java | 26 ++-
.../TestCapacitySchedulerPlanFollower.java | 3 +-
.../reservation/TestFairReservationSystem.java | 55 ++---
.../TestFairSchedulerPlanFollower.java | 203 +++
.../TestCapacitySchedulerDynamicBehavior.java | 3 +-
.../scheduler/fair/FairSchedulerTestBase.java | 4 +-
.../fair/TestFairSchedulerPreemption.java | 3 +-
19 files changed, 574 insertions(+), 59 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7adffad2/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c737a9a..8139c72 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -38,6 +38,8 @@ Release 2.7.0 - UNRELEASED
YARN-2738. [YARN-2574] Add FairReservationSystem for FairScheduler.
(Anubhav Dhoot via kasha)
+YARN-2881. [YARN-2574] Implement PlanFollower for FairScheduler.
+(Anubhav Dhoot via kasha)
IMPROVEMENTS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7adffad2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
index fa0835a..8a15ac6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
@@ -204,6 +204,8 @@ public abstract class AbstractReservationSystem extends
AbstractService
// currently only capacity scheduler is supported
if (scheduler instanceof CapacityScheduler) {
return CapacitySchedulerPlanFollower.class.getName();
+} else if (scheduler instanceof FairScheduler) {
+ return FairSchedulerPlanFollower.class.getName();
}
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7adffad2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
hadoop git commit: Move YARN-2360 from 2.6 to 2.7 in CHANGES.txt
Repository: hadoop Updated Branches: refs/heads/trunk 562a70194 - 41d72cbd4 Move YARN-2360 from 2.6 to 2.7 in CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41d72cbd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41d72cbd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41d72cbd Branch: refs/heads/trunk Commit: 41d72cbd48e6df7be3d177eaf04d73e88cf38381 Parents: 562a701 Author: Karthik Kambatla ka...@apache.org Authored: Tue Jan 6 03:31:11 2015 +0530 Committer: Karthik Kambatla ka...@apache.org Committed: Tue Jan 6 03:31:11 2015 +0530 -- hadoop-yarn-project/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/41d72cbd/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c464275..63faa60 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -161,6 +161,9 @@ Release 2.7.0 - UNRELEASED YARN-2998. Abstract out scheduler independent PlanFollower components. (Anubhav Dhoot via kasha) +YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the +scheduler page. (Ashwin Shankar and Wei Yan via kasha) + OPTIMIZATIONS @@ -554,9 +557,6 @@ Release 2.6.0 - 2014-11-18 YARN-2406. Move RM recovery related proto to yarn_server_resourcemanager_recovery.proto. (Tsuyoshi Ozawa via jianhe) -YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the -scheduler page. (Ashwin Shankar and Wei Yan via kasha) - YARN-1506. Changed RMNode/SchedulerNode to update resource with event notification. (Junping Du via jianhe)
[06/18] hadoop git commit: HADOOP-11455. KMS and Credential CLI should request confirmation for deletion by default. (Charles Lamb via yliu)
HADOOP-11455. KMS and Credential CLI should request confirmation for deletion
by default. (Charles Lamb via yliu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d8b51172
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d8b51172
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d8b51172
Branch: refs/heads/HDFS-EC
Commit: d8b51172cbbcd0647cc9bd6e57c98e651036cdc9
Parents: 0c94460
Author: yliu y...@apache.org
Authored: Mon Jan 5 06:55:08 2015 +0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:37 2015 -0800
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../org/apache/hadoop/crypto/key/KeyShell.java | 18 ++
.../hadoop/security/alias/CredentialShell.java | 20 +++-
.../apache/hadoop/crypto/key/TestKeyShell.java | 3 ++-
.../hadoop/security/alias/TestCredShell.java| 7 ---
5 files changed, 30 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8b51172/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index d7ebeac..ec75e8d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -452,6 +452,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11399. Java Configuration file and .xml files should be
automatically cross-compared (rchiang via rkanter)
+HADOOP-11455. KMS and Credential CLI should request confirmation for
+deletion by default. (Charles Lamb via yliu)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8b51172/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
index e0ca624..4c72d06 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyShell.java
@@ -47,7 +47,7 @@ public class KeyShell extends Configured implements Tool {
[ + ListCommand.USAGE + ]\n;
private static final String LIST_METADATA = keyShell.list.metadata;
- private boolean interactive = false;
+ private boolean interactive = true;
private Command command = null;
/** allows stdout to be captured if necessary */
@@ -169,8 +169,8 @@ public class KeyShell extends Configured implements Tool {
getConf().set(KeyProviderFactory.KEY_PROVIDER_PATH, args[++i]);
} else if (-metadata.equals(args[i])) {
getConf().setBoolean(LIST_METADATA, true);
- } else if (-i.equals(args[i]) || (-interactive.equals(args[i]))) {
-interactive = true;
+ } else if (-f.equals(args[i]) || (-force.equals(args[i]))) {
+interactive = false;
} else if (-help.equals(args[i])) {
printKeyShellUsage();
return 1;
@@ -367,11 +367,13 @@ public class KeyShell extends Configured implements Tool {
}
private class DeleteCommand extends Command {
-public static final String USAGE = delete keyname [-provider
provider] [-help];
+public static final String USAGE =
+delete keyname [-provider provider] [-f] [-help];
public static final String DESC =
The delete subcommand deletes all versions of the key\n +
specified by the keyname argument from within the\n +
-provider specified -provider.;
+provider specified -provider. The command asks for\n +
+user confirmation unless -f is specified.;
String keyName = null;
boolean cont = true;
@@ -397,10 +399,10 @@ public class KeyShell extends Configured implements Tool {
try {
cont = ToolRunner
.confirmPrompt(You are about to DELETE all versions of
- + key: + keyName + from KeyProvider
- + provider + . Continue?:);
+ + key + keyName + from KeyProvider
+ + provider + . Continue? );
if (!cont) {
-out.println(Nothing has been be deleted.);
+out.println(keyName + has not been deleted.);
}
return cont;
} catch (IOException e) {
[12/18] hadoop git commit: YARN-2922. ConcurrentModificationException in CapacityScheduler's LeafQueue. Contributed by Rohith Sharmaks.
YARN-2922. ConcurrentModificationException in CapacityScheduler's LeafQueue.
Contributed by Rohith Sharmaks.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c944600
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c944600
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c944600
Branch: refs/heads/HDFS-EC
Commit: 0c9446003fa9b462f75736d42c32925d931059c6
Parents: 10415a0
Author: Tsuyoshi Ozawa oz...@apache.org
Authored: Mon Jan 5 00:08:31 2015 +0900
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:37 2015 -0800
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/capacity/LeafQueue.java | 4 +-
.../scheduler/capacity/TestLeafQueue.java | 86
3 files changed, 91 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c944600/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e6694f1..0d33b4a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -308,6 +308,9 @@ Release 2.7.0 - UNRELEASED
YARN-2991. Fixed DrainDispatcher to reuse the draining code path in
AsyncDispatcher. (Rohith Sharmaks via zjshen)
+YARN-2922. ConcurrentModificationException in CapacityScheduler's
LeafQueue.
+(Rohith Sharmaks via ozawa)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c944600/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index f129ff4..47679a6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -1878,7 +1878,7 @@ public class LeafQueue extends AbstractCSQueue {
}
// return a single Resource capturing the overal amount of pending resources
- public Resource getTotalResourcePending() {
+ public synchronized Resource getTotalResourcePending() {
Resource ret = BuilderUtils.newResource(0, 0);
for (FiCaSchedulerApp f : activeApplications) {
Resources.addTo(ret, f.getTotalPendingRequests());
@@ -1887,7 +1887,7 @@ public class LeafQueue extends AbstractCSQueue {
}
@Override
- public void collectSchedulerApplications(
+ public synchronized void collectSchedulerApplications(
CollectionApplicationAttemptId apps) {
for (FiCaSchedulerApp pendingApp : pendingApplications) {
apps.add(pendingApp.getApplicationAttemptId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c944600/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index 642363e..fb7bb2c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -37,11 +37,13 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
+import java.util.ConcurrentModificationException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import
[18/18] hadoop git commit: Move YARN-2360 from 2.6 to 2.7 in CHANGES.txt
Move YARN-2360 from 2.6 to 2.7 in CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf023117 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf023117 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf023117 Branch: refs/heads/HDFS-EC Commit: cf023117983c6e7be95742af2bac1042fa539874 Parents: 35805a6 Author: Karthik Kambatla ka...@apache.org Authored: Tue Jan 6 03:31:11 2015 +0530 Committer: Zhe Zhang z...@apache.org Committed: Mon Jan 5 14:48:38 2015 -0800 -- hadoop-yarn-project/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf023117/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index c464275..63faa60 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -161,6 +161,9 @@ Release 2.7.0 - UNRELEASED YARN-2998. Abstract out scheduler independent PlanFollower components. (Anubhav Dhoot via kasha) +YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the +scheduler page. (Ashwin Shankar and Wei Yan via kasha) + OPTIMIZATIONS @@ -554,9 +557,6 @@ Release 2.6.0 - 2014-11-18 YARN-2406. Move RM recovery related proto to yarn_server_resourcemanager_recovery.proto. (Tsuyoshi Ozawa via jianhe) -YARN-2360. Fair Scheduler: Display dynamic fair share for queues on the -scheduler page. (Ashwin Shankar and Wei Yan via kasha) - YARN-1506. Changed RMNode/SchedulerNode to update resource with event notification. (Junping Du via jianhe)
[16/18] hadoop git commit: HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on Windows. Contributed by Arpit Agarwal.
HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on
Windows. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9a3a0697
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9a3a0697
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9a3a0697
Branch: refs/heads/HDFS-EC
Commit: 9a3a0697c2f6acac8de5fe3c85a6d7bddcc58c95
Parents: f3aced2
Author: cnauroth cnaur...@apache.org
Authored: Mon Jan 5 10:39:15 2015 -0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:38 2015 -0800
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java | 1 +
2 files changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a3a0697/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8117383..48e3695 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -639,6 +639,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7563. NFS gateway parseStaticMap NumberFormatException
(Yongjun Zhang via brandonli)
+HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on
+Windows. (Arpit Agarwal via cnauroth)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9a3a0697/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 49d3c6d..ad2197a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -633,6 +633,7 @@ public class TestLazyPersistFiles extends
LazyPersistTestCase {
LOG.info(Restarting the DataNode);
cluster.restartDataNode(0, true);
cluster.waitActive();
+triggerBlockReport();
// Ensure that the replica is now on persistent storage.
ensureFileReplicasOnStorageType(path1, DEFAULT);
hadoop git commit: YARN-2958. Made RMStateStore not update the last sequence number when updating the delegation token. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 60418c9b2 - c6cf74898
YARN-2958. Made RMStateStore not update the last sequence number when updating
the delegation token. Contributed by Varun Saxena.
(cherry picked from commit 562a701945be3a672f9cb5a52cc6db2c1589ba2b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6cf7489
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6cf7489
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6cf7489
Branch: refs/heads/branch-2
Commit: c6cf7489853d603bb1a77ad699b3068cb4779bbf
Parents: 60418c9
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Jan 5 13:33:07 2015 -0800
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Jan 5 13:35:09 2015 -0800
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../recovery/FileSystemRMStateStore.java| 53 +---
.../recovery/LeveldbRMStateStore.java | 45 +
.../recovery/MemoryRMStateStore.java| 32 +++-
.../recovery/NullRMStateStore.java | 12 ++---
.../resourcemanager/recovery/RMStateStore.java | 40 +++
.../recovery/RMStateStoreRMDTEvent.java | 10 +---
.../recovery/ZKRMStateStore.java| 40 +++
.../RMDelegationTokenSecretManager.java | 15 ++
.../recovery/RMStateStoreTestBase.java | 13 ++---
.../recovery/TestZKRMStateStore.java| 12 ++---
11 files changed, 131 insertions(+), 144 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6cf7489/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 06fcedc..83acc08 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -279,6 +279,9 @@ Release 2.7.0 - UNRELEASED
YARN-2922. ConcurrentModificationException in CapacityScheduler's
LeafQueue.
(Rohith Sharmaks via ozawa)
+YARN-2958. Made RMStateStore not update the last sequence number when
updating the
+delegation token. (Varun Saxena via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6cf7489/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 51e3916..7783662 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -60,8 +60,6 @@ import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AM
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-
import com.google.common.annotations.VisibleForTesting;
@Private
@@ -452,11 +450,10 @@ public class FileSystemRMStateStore extends RMStateStore {
}
@Override
- public synchronized void storeRMDelegationTokenAndSequenceNumberState(
- RMDelegationTokenIdentifier identifier, Long renewDate,
- int latestSequenceNumber) throws Exception {
-storeOrUpdateRMDelegationTokenAndSequenceNumberState(
-identifier, renewDate,latestSequenceNumber, false);
+ public synchronized void storeRMDelegationTokenState(
+ RMDelegationTokenIdentifier identifier, Long renewDate)
+ throws Exception {
+storeOrUpdateRMDelegationTokenState(identifier, renewDate, false);
}
@Override
@@ -469,16 +466,15 @@ public class FileSystemRMStateStore extends RMStateStore {
}
@Override
- protected void updateRMDelegationTokenAndSequenceNumberInternal(
- RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
- int latestSequenceNumber) throws
hadoop git commit: YARN-2958. Made RMStateStore not update the last sequence number when updating the delegation token. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/trunk dfd2589bc - 562a70194
YARN-2958. Made RMStateStore not update the last sequence number when updating
the delegation token. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/562a7019
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/562a7019
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/562a7019
Branch: refs/heads/trunk
Commit: 562a701945be3a672f9cb5a52cc6db2c1589ba2b
Parents: dfd2589
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Jan 5 13:33:07 2015 -0800
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Jan 5 13:33:07 2015 -0800
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../recovery/FileSystemRMStateStore.java| 53 +---
.../recovery/LeveldbRMStateStore.java | 45 +
.../recovery/MemoryRMStateStore.java| 32 +++-
.../recovery/NullRMStateStore.java | 12 ++---
.../resourcemanager/recovery/RMStateStore.java | 40 +++
.../recovery/RMStateStoreRMDTEvent.java | 10 +---
.../recovery/ZKRMStateStore.java| 40 +++
.../RMDelegationTokenSecretManager.java | 15 ++
.../recovery/RMStateStoreTestBase.java | 13 ++---
.../recovery/TestZKRMStateStore.java| 12 ++---
11 files changed, 131 insertions(+), 144 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562a7019/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0d33b4a..c464275 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -311,6 +311,9 @@ Release 2.7.0 - UNRELEASED
YARN-2922. ConcurrentModificationException in CapacityScheduler's
LeafQueue.
(Rohith Sharmaks via ozawa)
+YARN-2958. Made RMStateStore not update the last sequence number when
updating the
+delegation token. (Varun Saxena via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/562a7019/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 51e3916..7783662 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -60,8 +60,6 @@ import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AM
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-
import com.google.common.annotations.VisibleForTesting;
@Private
@@ -452,11 +450,10 @@ public class FileSystemRMStateStore extends RMStateStore {
}
@Override
- public synchronized void storeRMDelegationTokenAndSequenceNumberState(
- RMDelegationTokenIdentifier identifier, Long renewDate,
- int latestSequenceNumber) throws Exception {
-storeOrUpdateRMDelegationTokenAndSequenceNumberState(
-identifier, renewDate,latestSequenceNumber, false);
+ public synchronized void storeRMDelegationTokenState(
+ RMDelegationTokenIdentifier identifier, Long renewDate)
+ throws Exception {
+storeOrUpdateRMDelegationTokenState(identifier, renewDate, false);
}
@Override
@@ -469,16 +466,15 @@ public class FileSystemRMStateStore extends RMStateStore {
}
@Override
- protected void updateRMDelegationTokenAndSequenceNumberInternal(
- RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
- int latestSequenceNumber) throws Exception {
-storeOrUpdateRMDelegationTokenAndSequenceNumberState(
-
[08/18] hadoop git commit: YARN-2987. Fixed ClientRMService#getQueueInfo to check against queue and app ACLs. Contributed by Varun Saxena
YARN-2987. Fixed ClientRMService#getQueueInfo to check against queue and app
ACLs. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82add253
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82add253
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82add253
Branch: refs/heads/HDFS-EC
Commit: 82add253a5959c5321a587f4c1fc966186589fbd
Parents: d5a33b4
Author: Jian He jia...@apache.org
Authored: Tue Dec 30 17:15:37 2014 -0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:37 2015 -0800
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../server/resourcemanager/ClientRMService.java | 19 +++-
.../resourcemanager/TestClientRMService.java| 31 +++-
3 files changed, 51 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82add253/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0ebf8a3..f2115e1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -297,6 +297,9 @@ Release 2.7.0 - UNRELEASED
YARN-2938. Fixed new findbugs warnings in hadoop-yarn-resourcemanager and
hadoop-yarn-applicationhistoryservice. (Varun Saxena via zjshen)
+YARN-2987. Fixed ClientRMService#getQueueInfo to check against queue and
+app ACLs. (Varun Saxena via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82add253/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index bee6bf8..0c37eb9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -827,6 +827,14 @@ public class ClientRMService extends AbstractService
implements
@Override
public GetQueueInfoResponse getQueueInfo(GetQueueInfoRequest request)
throws YarnException {
+UserGroupInformation callerUGI;
+try {
+ callerUGI = UserGroupInformation.getCurrentUser();
+} catch (IOException ie) {
+ LOG.info(Error getting UGI , ie);
+ throw RPCUtil.getRemoteException(ie);
+}
+
GetQueueInfoResponse response =
recordFactory.newRecordInstance(GetQueueInfoResponse.class);
try {
@@ -841,7 +849,16 @@ public class ClientRMService extends AbstractService
implements
appReports = new ArrayListApplicationReport(apps.size());
for (ApplicationAttemptId app : apps) {
RMApp rmApp = rmContext.getRMApps().get(app.getApplicationId());
- appReports.add(rmApp.createAndGetApplicationReport(null, true));
+ if (rmApp != null) {
+// Check if user is allowed access to this app
+if (!checkAccess(callerUGI, rmApp.getUser(),
+ApplicationAccessType.VIEW_APP, rmApp)) {
+ continue;
+}
+appReports.add(
+rmApp.createAndGetApplicationReport(
+callerUGI.getUserName(), true));
+ }
}
}
queueInfo.setApplications(appReports);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82add253/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index a344e9a..a684346 100644
---
[02/18] hadoop git commit: Revert YARN-2492(wrong jira number). Added node-labels page on RM web UI. Contributed by Wangda Tan
Revert YARN-2492(wrong jira number). Added node-labels page on RM web UI.
Contributed by Wangda Tan
This reverts commit 5f57b904f550515693d93a2959e663b0d0260696.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/181827a5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/181827a5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/181827a5
Branch: refs/heads/HDFS-EC
Commit: 181827a5116259bd583b5350b919e8af2bff6db0
Parents: 756b66b
Author: Jian He jia...@apache.org
Authored: Tue Dec 30 16:46:26 2014 -0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:36 2015 -0800
--
hadoop-yarn-project/CHANGES.txt | 2 -
.../nodelabels/CommonNodeLabelsManager.java | 21 ++-
.../hadoop/yarn/nodelabels/NodeLabel.java | 96 -
.../hadoop/yarn/webapp/YarnWebParams.java | 1 -
.../nodelabels/RMNodeLabelsManager.java | 36 ++---
.../server/resourcemanager/webapp/NavBlock.java | 3 +-
.../resourcemanager/webapp/NodeLabelsPage.java | 91
.../resourcemanager/webapp/NodesPage.java | 139 +--
.../server/resourcemanager/webapp/RMWebApp.java | 1 -
.../resourcemanager/webapp/RmController.java| 6 +-
.../yarn/server/resourcemanager/MockNodes.java | 46 +-
.../TestWorkPreservingRMRestart.java| 2 +-
.../nodelabels/TestRMNodeLabelsManager.java | 33 -
.../resourcemanager/webapp/TestNodesPage.java | 45 --
14 files changed, 101 insertions(+), 421 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/181827a5/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0ebf8a3..e707ea7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -155,8 +155,6 @@ Release 2.7.0 - UNRELEASED
YARN-2993. Several fixes (missing acl check, error log msg ...) and some
refinement in AdminService. (Yi Liu via junping_du)
-YARN-2943. Added node-labels page on RM web UI. (Wangda Tan via jianhe)
-
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/181827a5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index e888cc5..070aa1f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -72,8 +72,8 @@ public class CommonNodeLabelsManager extends AbstractService {
protected Dispatcher dispatcher;
- protected ConcurrentMapString, NodeLabel labelCollections =
- new ConcurrentHashMapString, NodeLabel();
+ protected ConcurrentMapString, Label labelCollections =
+ new ConcurrentHashMapString, Label();
protected ConcurrentMapString, Host nodeCollections =
new ConcurrentHashMapString, Host();
@@ -82,6 +82,19 @@ public class CommonNodeLabelsManager extends AbstractService
{
protected NodeLabelsStore store;
+ protected static class Label {
+private Resource resource;
+
+protected Label() {
+ this.resource = Resource.newInstance(0, 0);
+}
+
+public Resource getResource() {
+ return this.resource;
+}
+
+ }
+
/**
* A codeHost/code can have multiple codeNode/codes
*/
@@ -188,7 +201,7 @@ public class CommonNodeLabelsManager extends
AbstractService {
protected void serviceInit(Configuration conf) throws Exception {
initNodeLabelStore(conf);
-labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL));
+labelCollections.put(NO_LABEL, new Label());
}
protected void initNodeLabelStore(Configuration conf) throws Exception {
@@ -258,7 +271,7 @@ public class CommonNodeLabelsManager extends
AbstractService {
for (String label : labels) {
// shouldn't overwrite it to avoid changing the Label.resource
if (this.labelCollections.get(label) == null) {
-this.labelCollections.put(label, new NodeLabel(label));
+this.labelCollections.put(label, new Label());
newLabels.add(label);
}
}
[04/18] hadoop git commit: MAPREDUCE-6149. Document override log4j.properties in MR job. Contributed by Junping Du.
MAPREDUCE-6149. Document override log4j.properties in MR job. Contributed by Junping Du. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5abc670b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5abc670b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5abc670b Branch: refs/heads/HDFS-EC Commit: 5abc670b76a11a2e1edff732822b7edfe09c1278 Parents: f123b71 Author: Harsh J ha...@cloudera.com Authored: Tue Dec 30 23:12:32 2014 +0530 Committer: Zhe Zhang z...@apache.org Committed: Mon Jan 5 14:48:36 2015 -0800 -- hadoop-mapreduce-project/CHANGES.txt | 3 +++ .../src/main/resources/mapred-default.xml| 15 +++ 2 files changed, 18 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5abc670b/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index 0a9ee8d..82295de 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -238,6 +238,9 @@ Release 2.7.0 - UNRELEASED IMPROVEMENTS +MAPREDUCE-6149. Document override log4j.properties in MR job. +(Junping Du via harsh) + MAPREDUCE-6194. Bubble up final exception in failures during creation of output collectors (Varun Saxena via harsh) http://git-wip-us.apache.org/repos/asf/hadoop/blob/5abc670b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml -- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 6e0deaa..30e291b 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -324,6 +324,8 @@ valueINFO/value descriptionThe logging level for the map task. The allowed levels are: OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL. + The setting here could be overridden if mapreduce.job.log4j-properties-file + is set. /description /property @@ -332,6 +334,8 @@ valueINFO/value descriptionThe logging level for the reduce task. The allowed levels are: OFF, FATAL, ERROR, WARN, INFO, DEBUG, TRACE and ALL. + The setting here could be overridden if mapreduce.job.log4j-properties-file + is set. /description /property @@ -1163,6 +1167,17 @@ /description /property + property +namemapreduce.job.log4j-properties-file/name +value/value +descriptionUsed to override the default settings of log4j in container-log4j.properties +for NodeManager. Like container-log4j.properties, it requires certain +framework appenders properly defined in this overriden file. The file on the +path will be added to distributed cache and classpath. If no-scheme is given +in the path, it defaults to point to a log4j file on the local FS. +/description + /property + property namemapreduce.job.end-notification.max.retry.interval/name value5000/value
[13/18] hadoop git commit: YARN-2958. Made RMStateStore not update the last sequence number when updating the delegation token. Contributed by Varun Saxena.
YARN-2958. Made RMStateStore not update the last sequence number when updating
the delegation token. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35805a69
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35805a69
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35805a69
Branch: refs/heads/HDFS-EC
Commit: 35805a695d81ec01880d18ed4ddf24f219c99c34
Parents: 9a3a069
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Jan 5 13:33:07 2015 -0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:38 2015 -0800
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../recovery/FileSystemRMStateStore.java| 53 +---
.../recovery/LeveldbRMStateStore.java | 45 +
.../recovery/MemoryRMStateStore.java| 32 +++-
.../recovery/NullRMStateStore.java | 12 ++---
.../resourcemanager/recovery/RMStateStore.java | 40 +++
.../recovery/RMStateStoreRMDTEvent.java | 10 +---
.../recovery/ZKRMStateStore.java| 40 +++
.../RMDelegationTokenSecretManager.java | 15 ++
.../recovery/RMStateStoreTestBase.java | 13 ++---
.../recovery/TestZKRMStateStore.java| 12 ++---
11 files changed, 131 insertions(+), 144 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35805a69/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0d33b4a..c464275 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -311,6 +311,9 @@ Release 2.7.0 - UNRELEASED
YARN-2922. ConcurrentModificationException in CapacityScheduler's
LeafQueue.
(Rohith Sharmaks via ozawa)
+YARN-2958. Made RMStateStore not update the last sequence number when
updating the
+delegation token. (Varun Saxena via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35805a69/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
index 51e3916..7783662 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
@@ -60,8 +60,6 @@ import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.AM
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationAttemptStateDataPBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.ApplicationStateDataPBImpl;
import
org.apache.hadoop.yarn.server.resourcemanager.recovery.records.impl.pb.EpochPBImpl;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-
import com.google.common.annotations.VisibleForTesting;
@Private
@@ -452,11 +450,10 @@ public class FileSystemRMStateStore extends RMStateStore {
}
@Override
- public synchronized void storeRMDelegationTokenAndSequenceNumberState(
- RMDelegationTokenIdentifier identifier, Long renewDate,
- int latestSequenceNumber) throws Exception {
-storeOrUpdateRMDelegationTokenAndSequenceNumberState(
-identifier, renewDate,latestSequenceNumber, false);
+ public synchronized void storeRMDelegationTokenState(
+ RMDelegationTokenIdentifier identifier, Long renewDate)
+ throws Exception {
+storeOrUpdateRMDelegationTokenState(identifier, renewDate, false);
}
@Override
@@ -469,16 +466,15 @@ public class FileSystemRMStateStore extends RMStateStore {
}
@Override
- protected void updateRMDelegationTokenAndSequenceNumberInternal(
- RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate,
- int latestSequenceNumber) throws Exception {
-storeOrUpdateRMDelegationTokenAndSequenceNumberState(
-rmDTIdentifier, renewDate,latestSequenceNumber, true);
+ protected void
[15/18] hadoop git commit: HADOOP-10908. Common needs updates for shell rewrite (aw)
HADOOP-10908. Common needs updates for shell rewrite (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e38cd055
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e38cd055
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e38cd055
Branch: refs/heads/HDFS-EC
Commit: e38cd055b3441d5c6c1bfbbc1a312c08fdf5f25b
Parents: cf02311
Author: Allen Wittenauer a...@apache.org
Authored: Mon Jan 5 14:26:41 2015 -0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:38 2015 -0800
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../src/site/apt/ClusterSetup.apt.vm| 348 ---
.../src/site/apt/CommandsManual.apt.vm | 316 +
.../src/site/apt/FileSystemShell.apt.vm | 313 ++---
.../src/site/apt/SingleCluster.apt.vm | 20 +-
5 files changed, 534 insertions(+), 465 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e38cd055/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0c76894..40e8d29 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -344,6 +344,8 @@ Trunk (Unreleased)
HADOOP-11397. Can't override HADOOP_IDENT_STRING (Kengo Seki via aw)
+HADOOP-10908. Common needs updates for shell rewrite (aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e38cd055/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
--
diff --git
a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
index f5f1deb..52b0552 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
@@ -11,83 +11,81 @@
~~ limitations under the License. See accompanying LICENSE file.
---
- Hadoop Map Reduce Next Generation-${project.version} - Cluster Setup
+ Hadoop ${project.version} - Cluster Setup
---
---
${maven.build.timestamp}
%{toc|section=1|fromDepth=0}
-Hadoop MapReduce Next Generation - Cluster Setup
+Hadoop Cluster Setup
* {Purpose}
- This document describes how to install, configure and manage non-trivial
+ This document describes how to install and configure
Hadoop clusters ranging from a few nodes to extremely large clusters
- with thousands of nodes.
+ with thousands of nodes. To play with Hadoop, you may first want to
+ install it on a single machine (see {{{./SingleCluster.html}Single Node
Setup}}).
- To play with Hadoop, you may first want to install it on a single
- machine (see {{{./SingleCluster.html}Single Node Setup}}).
+ This document does not cover advanced topics such as
{{{./SecureMode.html}Security}} or
+ High Availability.
* {Prerequisites}
- Download a stable version of Hadoop from Apache mirrors.
+ * Install Java. See the
{{{http://wiki.apache.org/hadoop/HadoopJavaVersions}Hadoop Wiki}} for known
good versions.
+ * Download a stable version of Hadoop from Apache mirrors.
* {Installation}
Installing a Hadoop cluster typically involves unpacking the software on all
- the machines in the cluster or installing RPMs.
+ the machines in the cluster or installing it via a packaging system as
+ appropriate for your operating system. It is important to divide up the
hardware
+ into functions.
Typically one machine in the cluster is designated as the NameNode and
- another machine the as ResourceManager, exclusively. These are the masters.
+ another machine the as ResourceManager, exclusively. These are the masters.
Other
+ services (such as Web App Proxy Server and MapReduce Job History server) are
usually
+ run either on dedicated hardware or on shared infrastrucutre, depending upon
the load.
The rest of the machines in the cluster act as both DataNode and NodeManager.
These are the slaves.
-* {Running Hadoop in Non-Secure Mode}
+* {Configuring Hadoop in Non-Secure Mode}
- The following sections describe how to configure a Hadoop cluster.
-
- {Configuration Files}
-
-Hadoop configuration is driven by two types of important configuration
files:
+Hadoop's Java configuration is driven by two types of important
configuration files:
* Read-only default configuration - core-default.xml,
hdfs-default.xml, yarn-default.xml and
mapred-default.xml.
- *
[07/18] hadoop git commit: YARN-2998. Abstract out scheduler independent PlanFollower components. (Anubhav Dhoot via kasha)
YARN-2998. Abstract out scheduler independent PlanFollower components. (Anubhav
Dhoot via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e28fa62
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e28fa62
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e28fa62
Branch: refs/heads/HDFS-EC
Commit: 1e28fa622d02863996901beb1ebc75b8270cab3d
Parents: 82add25
Author: Karthik Kambatla ka...@apache.org
Authored: Tue Dec 30 19:52:43 2014 -0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:37 2015 -0800
--
.../sls/scheduler/ResourceSchedulerWrapper.java | 6 +
hadoop-yarn-project/CHANGES.txt | 5 +
.../AbstractSchedulerPlanFollower.java | 412 +++
.../CapacitySchedulerPlanFollower.java | 361
.../scheduler/YarnScheduler.java| 5 +
.../scheduler/fair/FSLeafQueue.java | 4 +-
.../scheduler/fair/FairScheduler.java | 3 +-
.../scheduler/fifo/FifoScheduler.java | 5 +
.../reservation/ReservationSystemTestUtil.java | 4 +-
.../TestCapacitySchedulerPlanFollower.java | 209 +++---
.../TestSchedulerPlanFollowerBase.java | 191 +
11 files changed, 755 insertions(+), 450 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e28fa62/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
--
diff --git
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
index a65f776..0a80291 100644
---
a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
+++
b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
@@ -84,6 +84,7 @@ import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoSchedule
import org.apache.hadoop.yarn.sls.SLSRunner;
import org.apache.hadoop.yarn.sls.conf.SLSConfiguration;
import org.apache.hadoop.yarn.sls.web.SLSWebApp;
+import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.log4j.Logger;
@@ -867,6 +868,11 @@ final public class ResourceSchedulerWrapper
}
@Override
+ public ResourceCalculator getResourceCalculator() {
+return scheduler.getResourceCalculator();
+ }
+
+ @Override
public int getNumClusterNodes() {
return scheduler.getNumClusterNodes();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e28fa62/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f2115e1..01df44f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -70,6 +70,7 @@ Release 2.7.0 - UNRELEASED
YARN-2738. [YARN-2574] Add FairReservationSystem for FairScheduler.
(Anubhav Dhoot via kasha)
+
IMPROVEMENTS
YARN-2950. Change message to mandate, not suggest JS requirement on UI.
@@ -157,6 +158,10 @@ Release 2.7.0 - UNRELEASED
YARN-2943. Added node-labels page on RM web UI. (Wangda Tan via jianhe)
+YARN-2998. Abstract out scheduler independent PlanFollower components.
+(Anubhav Dhoot via kasha)
+
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e28fa62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
new file mode 100644
index 000..0de4dcf
--- /dev/null
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
@@ -0,0 +1,412 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding
[10/18] hadoop git commit: HDFS-7563. NFS gateway parseStaticMap NumberFormatException. Contributed by Yongjun Zhang
HDFS-7563. NFS gateway parseStaticMap NumberFormatException. Contributed by
Yongjun Zhang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de378cb5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de378cb5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de378cb5
Branch: refs/heads/HDFS-EC
Commit: de378cb59f00234839009b66eebc5b3b0040c30f
Parents: 1e28fa6
Author: Brandon Li brando...@apache.org
Authored: Fri Jan 2 10:49:50 2015 -0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:37 2015 -0800
--
.../java/org/apache/hadoop/security/ShellBasedIdMapping.java | 4 ++--
.../org/apache/hadoop/security/TestShellBasedIdMapping.java | 8 +++-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
3 files changed, 12 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de378cb5/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
index e995cb6..428e835 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
@@ -578,8 +578,8 @@ public class ShellBasedIdMapping implements
IdMappingServiceProvider {
// We know the line is fine to parse without error checking like this
// since it matched the regex above.
String firstComponent = lineMatcher.group(1);
-int remoteId = Integer.parseInt(lineMatcher.group(2));
-int localId = Integer.parseInt(lineMatcher.group(3));
+int remoteId = parseId(lineMatcher.group(2));
+int localId = parseId(lineMatcher.group(3));
if (firstComponent.equals(uid)) {
uidMapping.put(localId, remoteId);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de378cb5/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java
index ec8ac1d..857c706 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestShellBasedIdMapping.java
@@ -55,7 +55,9 @@ public class TestShellBasedIdMapping {
uid 13 302\n +
gid\t11\t201\n + // Tabs instead of spaces.
\n + // Entirely empty line.
-gid 12 202;
+gid 12 202\n +
+uid 4294967294 123\n +
+gid 4294967295 321;
OutputStream out = new FileOutputStream(tempStaticMapFile);
out.write(staticMapFileContents.getBytes());
out.close();
@@ -71,6 +73,10 @@ public class TestShellBasedIdMapping {
assertEquals(1, (int)parsedMap.uidMapping.get(10001));
// Ensure pass-through of unmapped IDs works.
assertEquals(1000, (int)parsedMap.uidMapping.get(1000));
+
+assertEquals(-2, (int)parsedMap.uidMapping.get(123));
+assertEquals(-1, (int)parsedMap.gidMapping.get(321));
+
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de378cb5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8525402..8117383 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -636,6 +636,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7456. De-duplicate AclFeature instances with same AclEntries do reduce
memory footprint of NameNode (vinayakumarb)
+HDFS-7563. NFS gateway parseStaticMap NumberFormatException
+(Yongjun Zhang via brandonli)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
[01/18] hadoop git commit: YARN-2493. Added node-labels page on RM web UI. Contributed by Wangda Tan
Repository: hadoop
Updated Branches:
refs/heads/HDFS-EC 43dd72d9b - e38cd055b
YARN-2493. Added node-labels page on RM web UI. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5a33b4a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5a33b4a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5a33b4a
Branch: refs/heads/HDFS-EC
Commit: d5a33b4acb03d7e27baf7e4cda91a0a1563941dd
Parents: 181827a
Author: Jian He jia...@apache.org
Authored: Tue Dec 30 15:35:45 2014 -0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:36 2015 -0800
--
hadoop-yarn-project/CHANGES.txt | 2 +
.../nodelabels/CommonNodeLabelsManager.java | 21 +--
.../hadoop/yarn/nodelabels/NodeLabel.java | 96 +
.../hadoop/yarn/webapp/YarnWebParams.java | 1 +
.../nodelabels/RMNodeLabelsManager.java | 36 +++--
.../server/resourcemanager/webapp/NavBlock.java | 3 +-
.../resourcemanager/webapp/NodeLabelsPage.java | 91
.../resourcemanager/webapp/NodesPage.java | 139 ++-
.../server/resourcemanager/webapp/RMWebApp.java | 1 +
.../resourcemanager/webapp/RmController.java| 6 +-
.../yarn/server/resourcemanager/MockNodes.java | 46 +-
.../TestWorkPreservingRMRestart.java| 2 +-
.../nodelabels/TestRMNodeLabelsManager.java | 33 +
.../resourcemanager/webapp/TestNodesPage.java | 45 ++
14 files changed, 421 insertions(+), 101 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5a33b4a/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e707ea7..0ebf8a3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -155,6 +155,8 @@ Release 2.7.0 - UNRELEASED
YARN-2993. Several fixes (missing acl check, error log msg ...) and some
refinement in AdminService. (Yi Liu via junping_du)
+YARN-2943. Added node-labels page on RM web UI. (Wangda Tan via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5a33b4a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index 070aa1f..e888cc5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -72,8 +72,8 @@ public class CommonNodeLabelsManager extends AbstractService {
protected Dispatcher dispatcher;
- protected ConcurrentMapString, Label labelCollections =
- new ConcurrentHashMapString, Label();
+ protected ConcurrentMapString, NodeLabel labelCollections =
+ new ConcurrentHashMapString, NodeLabel();
protected ConcurrentMapString, Host nodeCollections =
new ConcurrentHashMapString, Host();
@@ -82,19 +82,6 @@ public class CommonNodeLabelsManager extends AbstractService
{
protected NodeLabelsStore store;
- protected static class Label {
-private Resource resource;
-
-protected Label() {
- this.resource = Resource.newInstance(0, 0);
-}
-
-public Resource getResource() {
- return this.resource;
-}
-
- }
-
/**
* A codeHost/code can have multiple codeNode/codes
*/
@@ -201,7 +188,7 @@ public class CommonNodeLabelsManager extends
AbstractService {
protected void serviceInit(Configuration conf) throws Exception {
initNodeLabelStore(conf);
-labelCollections.put(NO_LABEL, new Label());
+labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL));
}
protected void initNodeLabelStore(Configuration conf) throws Exception {
@@ -271,7 +258,7 @@ public class CommonNodeLabelsManager extends
AbstractService {
for (String label : labels) {
// shouldn't overwrite it to avoid changing the Label.resource
if (this.labelCollections.get(label) == null) {
-this.labelCollections.put(label, new Label());
+this.labelCollections.put(label, new NodeLabel(label));
newLabels.add(label);
}
}
[03/18] hadoop git commit: YARN-2492. Added node-labels page on RM web UI. Contributed by Wangda Tan
YARN-2492. Added node-labels page on RM web UI. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/756b66bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/756b66bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/756b66bc
Branch: refs/heads/HDFS-EC
Commit: 756b66bcc38c3f4dcc878bf0e58f4a35cffeaea2
Parents: 5abc670
Author: Jian He jia...@apache.org
Authored: Tue Dec 30 15:35:45 2014 -0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:36 2015 -0800
--
hadoop-yarn-project/CHANGES.txt | 2 +
.../nodelabels/CommonNodeLabelsManager.java | 21 +--
.../hadoop/yarn/nodelabels/NodeLabel.java | 96 +
.../hadoop/yarn/webapp/YarnWebParams.java | 1 +
.../nodelabels/RMNodeLabelsManager.java | 36 +++--
.../server/resourcemanager/webapp/NavBlock.java | 3 +-
.../resourcemanager/webapp/NodeLabelsPage.java | 91
.../resourcemanager/webapp/NodesPage.java | 139 ++-
.../server/resourcemanager/webapp/RMWebApp.java | 1 +
.../resourcemanager/webapp/RmController.java| 6 +-
.../yarn/server/resourcemanager/MockNodes.java | 46 +-
.../TestWorkPreservingRMRestart.java| 2 +-
.../nodelabels/TestRMNodeLabelsManager.java | 33 +
.../resourcemanager/webapp/TestNodesPage.java | 45 ++
14 files changed, 421 insertions(+), 101 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/756b66bc/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e707ea7..0ebf8a3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -155,6 +155,8 @@ Release 2.7.0 - UNRELEASED
YARN-2993. Several fixes (missing acl check, error log msg ...) and some
refinement in AdminService. (Yi Liu via junping_du)
+YARN-2943. Added node-labels page on RM web UI. (Wangda Tan via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/756b66bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index 070aa1f..e888cc5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -72,8 +72,8 @@ public class CommonNodeLabelsManager extends AbstractService {
protected Dispatcher dispatcher;
- protected ConcurrentMapString, Label labelCollections =
- new ConcurrentHashMapString, Label();
+ protected ConcurrentMapString, NodeLabel labelCollections =
+ new ConcurrentHashMapString, NodeLabel();
protected ConcurrentMapString, Host nodeCollections =
new ConcurrentHashMapString, Host();
@@ -82,19 +82,6 @@ public class CommonNodeLabelsManager extends AbstractService
{
protected NodeLabelsStore store;
- protected static class Label {
-private Resource resource;
-
-protected Label() {
- this.resource = Resource.newInstance(0, 0);
-}
-
-public Resource getResource() {
- return this.resource;
-}
-
- }
-
/**
* A codeHost/code can have multiple codeNode/codes
*/
@@ -201,7 +188,7 @@ public class CommonNodeLabelsManager extends
AbstractService {
protected void serviceInit(Configuration conf) throws Exception {
initNodeLabelStore(conf);
-labelCollections.put(NO_LABEL, new Label());
+labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL));
}
protected void initNodeLabelStore(Configuration conf) throws Exception {
@@ -271,7 +258,7 @@ public class CommonNodeLabelsManager extends
AbstractService {
for (String label : labels) {
// shouldn't overwrite it to avoid changing the Label.resource
if (this.labelCollections.get(label) == null) {
-this.labelCollections.put(label, new Label());
+this.labelCollections.put(label, new NodeLabel(label));
newLabels.add(label);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/756b66bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java
[05/18] hadoop git commit: HADOOP-11039. ByteBufferReadable API doc is inconsistent with the implementations. (Yi Liu via Colin P. McCabe)
HADOOP-11039. ByteBufferReadable API doc is inconsistent with the
implementations. (Yi Liu via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f123b71c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f123b71c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f123b71c
Branch: refs/heads/HDFS-EC
Commit: f123b71cd7d4bb2f135c6265b753cdbbcd2dacfd
Parents: 43dd72d
Author: Colin Patrick Mccabe cmcc...@cloudera.com
Authored: Mon Dec 29 13:37:45 2014 -0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:36 2015 -0800
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/fs/ByteBufferReadable.java| 11 +--
2 files changed, 8 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f123b71c/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 4e25997..793e583 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -655,6 +655,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11448. Fix findbugs warnings in FileBasedIPList. (ozawa)
+HADOOP-11039. ByteBufferReadable API doc is inconsistent with the
+implementations. (Yi Liu via Colin P. McCabe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f123b71c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
index e18a115..aa6e85e 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/ByteBufferReadable.java
@@ -29,9 +29,8 @@ public interface ByteBufferReadable {
* Reads up to buf.remaining() bytes into buf. Callers should use
* buf.limit(..) to control the size of the desired read.
* p/
- * After a successful call, buf.position() and buf.limit() should be
- * unchanged, and therefore any data can be immediately read from buf.
- * buf.mark() may be cleared or updated.
+ * After a successful call, buf.position() will be advanced by the number
+ * of bytes read and buf.limit() should be unchanged.
* p/
* In the case of an exception, the values of buf.position() and buf.limit()
* are undefined, and callers should be prepared to recover from this
@@ -45,9 +44,9 @@ public interface ByteBufferReadable {
* signal an error upon their receipt.
*
* @param buf
- * the ByteBuffer to receive the results of the read operation. Up
to
- * buf.limit() - buf.position() bytes may be read.
- * @return the number of bytes available to read from buf
+ * the ByteBuffer to receive the results of the read operation.
+ * @return the number of bytes read, possibly zero, or -1 if
+ * reach end-of-stream
* @throws IOException
* if there is some error performing the read
*/
[17/18] hadoop git commit: HADOOP-11390 Metrics 2 ganglia provider to include hostname in unresolved address problems. (Varun Saxena via stevel)
HADOOP-11390 Metrics 2 ganglia provider to include hostname in unresolved
address problems. (Varun Saxena via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3aced27
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3aced27
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3aced27
Branch: refs/heads/HDFS-EC
Commit: f3aced275867384cf18be5c1e1b520e88c6aee5d
Parents: e9e00ac
Author: Steve Loughran ste...@apache.org
Authored: Mon Jan 5 16:09:25 2015 +
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:38 2015 -0800
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java | 7 +++
2 files changed, 10 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3aced27/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index baf68d6..0c76894 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -455,6 +455,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11455. KMS and Credential CLI should request confirmation for
deletion by default. (Charles Lamb via yliu)
+HADOOP-11390 Metrics 2 ganglia provider to include hostname in
+unresolved address problems. (Varun Saxena via stevel)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3aced27/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
index 164ea08..88ad647 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.metrics2.sink.ganglia;
import java.io.IOException;
import java.net.DatagramPacket;
import java.net.DatagramSocket;
+import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.net.SocketException;
import java.net.UnknownHostException;
@@ -257,6 +258,12 @@ public abstract class AbstractGangliaSink implements
MetricsSink {
protected void emitToGangliaHosts() throws IOException {
try {
for (SocketAddress socketAddress : metricsServers) {
+if (socketAddress == null || !(socketAddress instanceof
InetSocketAddress))
+ throw new IllegalArgumentException(Unsupported Address type);
+InetSocketAddress inetAddress = (InetSocketAddress)socketAddress;
+if(inetAddress.isUnresolved()) {
+ throw new UnknownHostException(Unresolved host: + inetAddress);
+}
DatagramPacket packet =
new DatagramPacket(buffer, offset, socketAddress);
datagramSocket.send(packet);
[09/18] hadoop git commit: HADOOP-10788. Rewrite kms to use new shell framework (John Smith via aw)
HADOOP-10788. Rewrite kms to use new shell framework (John Smith via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80f386d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80f386d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80f386d1
Branch: refs/heads/HDFS-EC
Commit: 80f386d1e50a4158c351e5518b6998f871b1383d
Parents: de378cb
Author: Allen Wittenauer a...@apache.org
Authored: Fri Jan 2 10:52:23 2015 -0800
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:37 2015 -0800
--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../src/main/bin/hadoop-functions.sh| 19 ++
.../hadoop-kms/src/main/conf/kms-env.sh | 45 +++-
.../hadoop-kms/src/main/libexec/kms-config.sh | 207 ---
.../hadoop-kms/src/main/sbin/kms.sh | 119 ++-
5 files changed, 171 insertions(+), 221 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80f386d1/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 793e583..d7ebeac 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -141,6 +141,8 @@ Trunk (Unreleased)
HADOOP-11352 Clean up test-patch.sh to disable +1 contrib tests
(Akira AJISAKA via stevel)
+HADOOP-10788. Rewrite kms to use new shell framework (John Smith via aw)
+
BUG FIXES
HADOOP-9451. Fault single-layer config if node group topology is enabled.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80f386d1/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index a953e0c..799aad0 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -666,6 +666,25 @@ function hadoop_finalize_classpath
hadoop_add_to_classpath_userpath
}
+function hadoop_finalize_catalina_opts
+{
+
+ local prefix=${HADOOP_CATALINA_PREFIX}
+
+ hadoop_add_param CATALINA_OPTS hadoop.home.dir
-Dhadoop.home.dir=${HADOOP_PREFIX}
+ if [[ -n ${JAVA_LIBRARY_PATH} ]]; then
+hadoop_add_param CATALINA_OPTS java.library.path
-Djava.library.path=${JAVA_LIBRARY_PATH}
+ fi
+ hadoop_add_param CATALINA_OPTS ${prefix}.home.dir
-D${prefix}.home.dir=${HADOOP_PREFIX}
+ hadoop_add_param CATALINA_OPTS ${prefix}.config.dir
-D${prefix}.config.dir=${HADOOP_CATALINA_CONFIG}
+ hadoop_add_param CATALINA_OPTS ${prefix}.log.dir
-D${prefix}.log.dir=${HADOOP_CATALINA_LOG}
+ hadoop_add_param CATALINA_OPTS ${prefix}.temp.dir
-D${prefix}.temp.dir=${HADOOP_CATALINA_TEMP}
+ hadoop_add_param CATALINA_OPTS ${prefix}.admin.port
-D${prefix}.admin.port=${HADOOP_CATALINA_ADMIN_PORT}
+ hadoop_add_param CATALINA_OPTS ${prefix}.http.port
-D${prefix}.http.port=${HADOOP_CATALINA_HTTP_PORT}
+ hadoop_add_param CATALINA_OPTS ${prefix}.max.threads
-D${prefix}.max.threads=${HADOOP_CATALINA_MAX_THREADS}
+ hadoop_add_param CATALINA_OPTS ${prefix}.ssl.keystore.file
-D${prefix}.ssl.keystore.file=${HADOOP_CATALINA_SSL_KEYSTORE_FILE}
+}
+
function hadoop_finalize
{
# user classpath gets added at the last minute. this allows
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80f386d1/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
--
diff --git a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
index 44dfe6a..7045379 100644
--- a/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
+++ b/hadoop-common-project/hadoop-kms/src/main/conf/kms-env.sh
@@ -14,19 +14,17 @@
#
# Set kms specific environment variables here.
-
-# Settings for the Embedded Tomcat that runs KMS
-# Java System properties for KMS should be specified in this variable
#
-# export CATALINA_OPTS=
+# hadoop-env.sh is read prior to this file.
+#
# KMS logs directory
#
-# export KMS_LOG=${KMS_HOME}/logs
+# export KMS_LOG=${HADOOP_LOG_DIR}
# KMS temporary directory
#
-# export KMS_TEMP=${KMS_HOME}/temp
+# export KMS_TEMP=${HADOOP_PREFIX}/temp
# The HTTP port used by KMS
#
@@ -34,7 +32,7 @@
# The Admin port used by KMS
#
-# export KMS_ADMIN_PORT=`expr ${KMS_HTTP_PORT} + 1`
+# export KMS_ADMIN_PORT=$((KMS_HTTP_PORT + 1))
# The maximum number of Tomcat handler threads
#
@@ -44,12 +42,37 @@
#
# export
[14/18] hadoop git commit: HADOOP-11446 S3AOutputStream should use shared thread pool to avoid OutOfMemoryError
HADOOP-11446 S3AOutputStream should use shared thread pool to avoid
OutOfMemoryError
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e9e00ac6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e9e00ac6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e9e00ac6
Branch: refs/heads/HDFS-EC
Commit: e9e00ac6e81d7e61372d82af948df73d78fe67b9
Parents: d8b5117
Author: Steve Loughran ste...@apache.org
Authored: Mon Jan 5 12:59:48 2015 +
Committer: Zhe Zhang z...@apache.org
Committed: Mon Jan 5 14:48:38 2015 -0800
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/fs/s3a/Constants.java | 17
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 91 ++--
.../apache/hadoop/fs/s3a/S3AOutputStream.java | 13 +--
4 files changed, 109 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e00ac6/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index ec75e8d..baf68d6 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -663,6 +663,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11039. ByteBufferReadable API doc is inconsistent with the
implementations. (Yi Liu via Colin P. McCabe)
+HADOOP-11446. S3AOutputStream should use shared thread pool to
+avoid OutOfMemoryError. (Ted Yu via stevel)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e00ac6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index ee4bf68..f1b5d3d 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -41,6 +41,23 @@ public class Constants {
public static final String MAX_PAGING_KEYS = fs.s3a.paging.maximum;
public static final int DEFAULT_MAX_PAGING_KEYS = 5000;
+ // the maximum number of threads to allow in the pool used by TransferManager
+ public static final String MAX_THREADS = fs.s3a.threads.max;
+ public static final int DEFAULT_MAX_THREADS = 256;
+
+ // the number of threads to keep in the pool used by TransferManager
+ public static final String CORE_THREADS = fs.s3a.threads.core;
+ public static final int DEFAULT_CORE_THREADS = DEFAULT_MAXIMUM_CONNECTIONS;
+
+ // when the number of threads is greater than the core, this is the maximum
time
+ // that excess idle threads will wait for new tasks before terminating.
+ public static final String KEEPALIVE_TIME = fs.s3a.threads.keepalivetime;
+ public static final int DEFAULT_KEEPALIVE_TIME = 60;
+
+ // the maximum number of tasks that the LinkedBlockingQueue can hold
+ public static final String MAX_TOTAL_TASKS = fs.s3a.max.total.tasks;
+ public static final int DEFAULT_MAX_TOTAL_TASKS = 1000;
+
// size of each of or multipart pieces in bytes
public static final String MULTIPART_SIZE = fs.s3a.multipart.size;
public static final long DEFAULT_MULTIPART_SIZE = 104857600; // 100 MB
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e00ac6/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
--
diff --git
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 457351d..e6b1557 100644
---
a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++
b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -26,6 +26,11 @@ import java.net.URI;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.s3.S3Credentials;
@@ -77,6 +82,7 @@ public class S3AFileSystem extends FileSystem {
private String bucket;
private int maxKeys;
private long partSize;
+ private TransferManager transfers;
private int partSizeThreshold;
public static final Logger LOG =
hadoop git commit: YARN-2881. [YARN-2574] Implement PlanFollower for FairScheduler. (Anubhav Dhoot via kasha)
Repository: hadoop
Updated Branches:
refs/heads/trunk 53caeaa16 - 0c4b11267
YARN-2881. [YARN-2574] Implement PlanFollower for FairScheduler. (Anubhav Dhoot
via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c4b1126
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c4b1126
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c4b1126
Branch: refs/heads/trunk
Commit: 0c4b11267717eb451fa6ed4c586317f2db32fbd5
Parents: 53caeaa
Author: Karthik Kambatla ka...@apache.org
Authored: Tue Jan 6 04:41:45 2015 +0530
Committer: Karthik Kambatla ka...@apache.org
Committed: Tue Jan 6 04:42:10 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 2 +
.../reservation/AbstractReservationSystem.java | 2 +
.../AbstractSchedulerPlanFollower.java | 3 +-
.../reservation/FairSchedulerPlanFollower.java | 141 +
.../reservation/ReservationConstants.java | 28 +++
.../scheduler/capacity/CapacityScheduler.java | 6 +-
.../scheduler/capacity/PlanQueue.java | 2 -
.../scheduler/fair/AllocationConfiguration.java | 14 ++
.../scheduler/fair/FSLeafQueue.java | 10 +
.../scheduler/fair/FairScheduler.java | 104 +-
.../scheduler/fair/QueueManager.java| 13 +-
.../fair/ReservationQueueConfiguration.java | 11 +
.../reservation/ReservationSystemTestUtil.java | 26 ++-
.../TestCapacitySchedulerPlanFollower.java | 3 +-
.../reservation/TestFairReservationSystem.java | 55 ++---
.../TestFairSchedulerPlanFollower.java | 203 +++
.../TestCapacitySchedulerDynamicBehavior.java | 3 +-
.../scheduler/fair/FairSchedulerTestBase.java | 4 +-
.../fair/TestFairSchedulerPreemption.java | 3 +-
19 files changed, 574 insertions(+), 59 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c4b1126/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 63faa60..12c8066 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -70,6 +70,8 @@ Release 2.7.0 - UNRELEASED
YARN-2738. [YARN-2574] Add FairReservationSystem for FairScheduler.
(Anubhav Dhoot via kasha)
+YARN-2881. [YARN-2574] Implement PlanFollower for FairScheduler.
+(Anubhav Dhoot via kasha)
IMPROVEMENTS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c4b1126/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
index fa0835a..8a15ac6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
@@ -204,6 +204,8 @@ public abstract class AbstractReservationSystem extends
AbstractService
// currently only capacity scheduler is supported
if (scheduler instanceof CapacityScheduler) {
return CapacitySchedulerPlanFollower.class.getName();
+} else if (scheduler instanceof FairScheduler) {
+ return FairSchedulerPlanFollower.class.getName();
}
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c4b1126/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractSchedulerPlanFollower.java
index 0de4dcf..ea7f27d 100644
---
hadoop git commit: HADOOP-11402. Negative user-to-group cache entries are never cleared for never-again-accessed users. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d070597a0 - 0c2d996c2
HADOOP-11402. Negative user-to-group cache entries are never cleared for
never-again-accessed users. Contributed by Varun Saxena.
(cherry picked from commit 53caeaa16b1450b54e994c77f5d0c8a767b88d57)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c2d996c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c2d996c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c2d996c
Branch: refs/heads/branch-2
Commit: 0c2d996c2ce2f3e1c959a04f15bb6fb0ba9c4b1f
Parents: d070597
Author: Benoy Antony be...@apache.org
Authored: Mon Jan 5 15:06:46 2015 -0800
Committer: Benoy Antony be...@apache.org
Committed: Mon Jan 5 15:09:19 2015 -0800
--
.../java/org/apache/hadoop/security/Groups.java | 36 +--
.../hadoop/security/TestGroupsCaching.java | 48
2 files changed, 71 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c2d996c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index f3c5094..9fd39b0 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -23,12 +23,14 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
+import java.util.Set;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Ticker;
import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.Cache;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import org.apache.hadoop.HadoopIllegalArgumentException;
@@ -60,14 +62,13 @@ public class Groups {
private final GroupMappingServiceProvider impl;
private final LoadingCacheString, ListString cache;
- private final ConcurrentHashMapString, Long negativeCacheMask =
-new ConcurrentHashMapString, Long();
private final MapString, ListString staticUserToGroupsMap =
new HashMapString, ListString();
private final long cacheTimeout;
private final long negativeCacheTimeout;
private final long warningDeltaMs;
private final Timer timer;
+ private SetString negativeCache;
public Groups(Configuration conf) {
this(conf, new Timer());
@@ -99,11 +100,24 @@ public class Groups {
.expireAfterWrite(10 * cacheTimeout, TimeUnit.MILLISECONDS)
.build(new GroupCacheLoader());
+if(negativeCacheTimeout 0) {
+ CacheString, Boolean tempMap = CacheBuilder.newBuilder()
+.expireAfterWrite(negativeCacheTimeout, TimeUnit.MILLISECONDS)
+.ticker(new TimerToTickerAdapter(timer))
+.build();
+ negativeCache = Collections.newSetFromMap(tempMap.asMap());
+}
+
if(LOG.isDebugEnabled())
LOG.debug(Group mapping impl= + impl.getClass().getName() +
; cacheTimeout= + cacheTimeout + ; warningDeltaMs= +
warningDeltaMs);
}
+
+ @VisibleForTesting
+ SetString getNegativeCache() {
+return negativeCache;
+ }
/*
* Parse the hadoop.user.group.static.mapping.overrides configuration to
@@ -159,13 +173,8 @@ public class Groups {
// Check the negative cache first
if (isNegativeCacheEnabled()) {
- Long expirationTime = negativeCacheMask.get(user);
- if (expirationTime != null) {
-if (timer.monotonicNow() expirationTime) {
- throw noGroupsForUser(user);
-} else {
- negativeCacheMask.remove(user, expirationTime);
-}
+ if (negativeCache.contains(user)) {
+throw noGroupsForUser(user);
}
}
@@ -212,8 +221,7 @@ public class Groups {
if (groups.isEmpty()) {
if (isNegativeCacheEnabled()) {
- long expirationTime = timer.monotonicNow() + negativeCacheTimeout;
- negativeCacheMask.put(user, expirationTime);
+ negativeCache.add(user);
}
// We throw here to prevent Cache from retaining an empty group
@@ -252,7 +260,9 @@ public class Groups {
LOG.warn(Error refreshing groups cache, e);
}
cache.invalidateAll();
-negativeCacheMask.clear();
+if(isNegativeCacheEnabled()) {
+ negativeCache.clear();
+}
}
/**
hadoop git commit: HADOOP-11459. Fix recent findbugs in ActiveStandbyElector, NetUtils and ShellBasedIdMapping (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 7adffad2b - 383e1c5f7
HADOOP-11459. Fix recent findbugs in ActiveStandbyElector, NetUtils and
ShellBasedIdMapping (Contributed by Vinayakumar B)
(cherry picked from commit 9803ae374f69942aec82ec6eeeb9722523a1ade0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/383e1c5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/383e1c5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/383e1c5f
Branch: refs/heads/branch-2
Commit: 383e1c5f737f84fad658e5c7aed48be81c207f38
Parents: 7adffad
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Jan 6 09:15:23 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Jan 6 09:16:44 2015 +0530
--
hadoop-common-project/hadoop-common/CHANGES.txt | 5 -
.../main/java/org/apache/hadoop/ha/ActiveStandbyElector.java| 4 +++-
.../src/main/java/org/apache/hadoop/net/NetUtils.java | 2 +-
.../java/org/apache/hadoop/security/ShellBasedIdMapping.java| 2 +-
4 files changed, 9 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/383e1c5f/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 7a57ef0..f0a1aae 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -318,7 +318,10 @@ Release 2.7.0 - UNRELEASED
HADOOP-11446. S3AOutputStream should use shared thread pool to
avoid OutOfMemoryError. (Ted Yu via stevel)
-
+
+HADOOP-11459. Fix recent findbugs in ActiveStandbyElector, NetUtils
+and ShellBasedIdMapping (vinayakumarb)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/383e1c5f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index 123f119..947baa9 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -1064,7 +1064,9 @@ public class ActiveStandbyElector implements
StatCallback, StringCallback {
public void process(WatchedEvent event) {
hasReceivedEvent.countDown();
try {
-hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS);
+if (!hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS)) {
+ LOG.debug(Event received with stale zk);
+}
ActiveStandbyElector.this.processWatchEvent(
zk, event);
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/383e1c5f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index ffee402..ef1092b 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -288,7 +288,7 @@ public class NetUtils {
try {
fqHost = SecurityUtil.getByName(host).getHostName();
// slight race condition, but won't hurt
-canonicalizedHostCache.put(host, fqHost);
+canonicalizedHostCache.putIfAbsent(host, fqHost);
} catch (UnknownHostException e) {
fqHost = host;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/383e1c5f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
index 428e835..28ab248 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
@@ -290,7 +290,7 @@ public
hadoop git commit: HDFS-7583. Fix findbug in TransferFsImage.java (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/trunk 9803ae374 - 4cd66f7fb
HDFS-7583. Fix findbug in TransferFsImage.java (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4cd66f7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4cd66f7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4cd66f7f
Branch: refs/heads/trunk
Commit: 4cd66f7fb280e53e2d398a62e922a8d68d150679
Parents: 9803ae3
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Jan 6 09:31:35 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Jan 6 09:31:35 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java | 5 +++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cd66f7f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 48e3695..e9d6ad2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -642,6 +642,8 @@ Release 2.7.0 - UNRELEASED
HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on
Windows. (Arpit Agarwal via cnauroth)
+HDFS-7583. Fix findbug in TransferFsImage.java (vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cd66f7f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
index 1f52ff7..c1e9d7f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
@@ -574,11 +574,12 @@ public class TransferFsImage {
LOG.info(Deleting temporary files: + files);
for (File file : files) {
- file.delete(); // ignore the return value
+ if (!file.delete()) {
+LOG.warn(Deleting + file + has failed);
+ }
}
}
-
private static MD5Hash parseMD5Header(HttpURLConnection connection) {
String header = connection.getHeaderField(MD5_HEADER);
return (header != null) ? new MD5Hash(header) : null;
hadoop git commit: HDFS-7583. Fix findbug in TransferFsImage.java (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 383e1c5f7 - 53ecb6358
HDFS-7583. Fix findbug in TransferFsImage.java (Contributed by Vinayakumar B)
(cherry picked from commit 4cd66f7fb280e53e2d398a62e922a8d68d150679)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/53ecb635
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/53ecb635
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/53ecb635
Branch: refs/heads/branch-2
Commit: 53ecb63587f64ea94f9d1bae9b339d09059e727a
Parents: 383e1c5
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Jan 6 09:31:35 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Jan 6 09:32:56 2015 +0530
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java | 5 +++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53ecb635/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index c0296fb..6561fb2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -385,6 +385,8 @@ Release 2.7.0 - UNRELEASED
HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on
Windows. (Arpit Agarwal via cnauroth)
+HDFS-7583. Fix findbug in TransferFsImage.java (vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/53ecb635/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
index 1f52ff7..c1e9d7f 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
@@ -574,11 +574,12 @@ public class TransferFsImage {
LOG.info(Deleting temporary files: + files);
for (File file : files) {
- file.delete(); // ignore the return value
+ if (!file.delete()) {
+LOG.warn(Deleting + file + has failed);
+ }
}
}
-
private static MD5Hash parseMD5Header(HttpURLConnection connection) {
String header = connection.getHeaderField(MD5_HEADER);
return (header != null) ? new MD5Hash(header) : null;
hadoop git commit: HADOOP-11459. Fix recent findbugs in ActiveStandbyElector, NetUtils and ShellBasedIdMapping (Contributed by Vinayakumar B)
Repository: hadoop
Updated Branches:
refs/heads/trunk 0c4b11267 - 9803ae374
HADOOP-11459. Fix recent findbugs in ActiveStandbyElector, NetUtils and
ShellBasedIdMapping (Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9803ae37
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9803ae37
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9803ae37
Branch: refs/heads/trunk
Commit: 9803ae374f69942aec82ec6eeeb9722523a1ade0
Parents: 0c4b112
Author: Vinayakumar B vinayakum...@apache.org
Authored: Tue Jan 6 09:15:23 2015 +0530
Committer: Vinayakumar B vinayakum...@apache.org
Committed: Tue Jan 6 09:15:23 2015 +0530
--
hadoop-common-project/hadoop-common/CHANGES.txt | 5 -
.../main/java/org/apache/hadoop/ha/ActiveStandbyElector.java| 4 +++-
.../src/main/java/org/apache/hadoop/net/NetUtils.java | 2 +-
.../java/org/apache/hadoop/security/ShellBasedIdMapping.java| 2 +-
4 files changed, 9 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9803ae37/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 40e8d29..4ceb77d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -670,7 +670,10 @@ Release 2.7.0 - UNRELEASED
HADOOP-11446. S3AOutputStream should use shared thread pool to
avoid OutOfMemoryError. (Ted Yu via stevel)
-
+
+HADOOP-11459. Fix recent findbugs in ActiveStandbyElector, NetUtils
+and ShellBasedIdMapping (vinayakumarb)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9803ae37/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index 123f119..947baa9 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -1064,7 +1064,9 @@ public class ActiveStandbyElector implements
StatCallback, StringCallback {
public void process(WatchedEvent event) {
hasReceivedEvent.countDown();
try {
-hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS);
+if (!hasSetZooKeeper.await(zkSessionTimeout, TimeUnit.MILLISECONDS)) {
+ LOG.debug(Event received with stale zk);
+}
ActiveStandbyElector.this.processWatchEvent(
zk, event);
} catch (Throwable t) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9803ae37/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index ffee402..ef1092b 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -288,7 +288,7 @@ public class NetUtils {
try {
fqHost = SecurityUtil.getByName(host).getHostName();
// slight race condition, but won't hurt
-canonicalizedHostCache.put(host, fqHost);
+canonicalizedHostCache.putIfAbsent(host, fqHost);
} catch (UnknownHostException e) {
fqHost = host;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9803ae37/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
index 428e835..28ab248 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/ShellBasedIdMapping.java
@@ -290,7 +290,7 @@ public class ShellBasedIdMapping implements
IdMappingServiceProvider {
return
hadoop git commit: HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on Windows. Contributed by Arpit Agarwal.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 4446512e9 - 60418c9b2
HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on
Windows. Contributed by Arpit Agarwal.
(cherry picked from commit dfd2589bcb0e83f073eab30e32badcf2e9f75a62)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/60418c9b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/60418c9b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/60418c9b
Branch: refs/heads/branch-2
Commit: 60418c9b2d5424337d1ba74e199a2993ca0c14a1
Parents: 4446512
Author: cnauroth cnaur...@apache.org
Authored: Mon Jan 5 10:39:15 2015 -0800
Committer: cnauroth cnaur...@apache.org
Committed: Mon Jan 5 10:39:38 2015 -0800
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java | 1 +
2 files changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60418c9b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b0dcef5..c0296fb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -382,6 +382,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7563. NFS gateway parseStaticMap NumberFormatException
(Yongjun Zhang via brandonli)
+HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on
+Windows. (Arpit Agarwal via cnauroth)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/60418c9b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 49d3c6d..ad2197a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -633,6 +633,7 @@ public class TestLazyPersistFiles extends
LazyPersistTestCase {
LOG.info(Restarting the DataNode);
cluster.restartDataNode(0, true);
cluster.waitActive();
+triggerBlockReport();
// Ensure that the replica is now on persistent storage.
ensureFileReplicasOnStorageType(path1, DEFAULT);
hadoop git commit: HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on Windows. Contributed by Arpit Agarwal.
Repository: hadoop
Updated Branches:
refs/heads/trunk 2403634de - dfd2589bc
HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on
Windows. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dfd2589b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dfd2589b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dfd2589b
Branch: refs/heads/trunk
Commit: dfd2589bcb0e83f073eab30e32badcf2e9f75a62
Parents: 2403634
Author: cnauroth cnaur...@apache.org
Authored: Mon Jan 5 10:39:15 2015 -0800
Committer: cnauroth cnaur...@apache.org
Committed: Mon Jan 5 10:39:15 2015 -0800
--
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java | 1 +
2 files changed, 4 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd2589b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
--
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8117383..48e3695 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -639,6 +639,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7563. NFS gateway parseStaticMap NumberFormatException
(Yongjun Zhang via brandonli)
+HDFS-7572. TestLazyPersistFiles#testDnRestartWithSavedReplicas is flaky on
+Windows. (Arpit Agarwal via cnauroth)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfd2589b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
--
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
index 49d3c6d..ad2197a 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestLazyPersistFiles.java
@@ -633,6 +633,7 @@ public class TestLazyPersistFiles extends
LazyPersistTestCase {
LOG.info(Restarting the DataNode);
cluster.restartDataNode(0, true);
cluster.waitActive();
+triggerBlockReport();
// Ensure that the replica is now on persistent storage.
ensureFileReplicasOnStorageType(path1, DEFAULT);
[1/2] hadoop git commit: HADOOP-11390 Metrics 2 ganglia provider to include hostname in unresolved address problems. (Varun Saxena via stevel)
Repository: hadoop
Updated Branches:
refs/heads/branch-2 2800e73c2 - 4446512e9
refs/heads/trunk 27d839586 - 2403634de
HADOOP-11390 Metrics 2 ganglia provider to include hostname in unresolved
address problems. (Varun Saxena via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4446512e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4446512e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4446512e
Branch: refs/heads/branch-2
Commit: 4446512e9e02ac8c900c444c79df58bb63319f4d
Parents: 2800e73
Author: Steve Loughran ste...@apache.org
Authored: Mon Jan 5 16:09:25 2015 +
Committer: Steve Loughran ste...@apache.org
Committed: Mon Jan 5 16:09:25 2015 +
--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java | 7 +++
2 files changed, 10 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4446512e/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 451ce61..7a57ef0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -100,6 +100,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11455. KMS and Credential CLI should request confirmation for
deletion by default. (Charles Lamb via yliu)
+HADOOP-11390 Metrics 2 ganglia provider to include hostname in
+unresolved address problems. (Varun Saxena via stevel)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4446512e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
index 15eea61..6757068 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/metrics2/sink/ganglia/AbstractGangliaSink.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.metrics2.sink.ganglia;
import java.io.IOException;
import java.net.DatagramPacket;
import java.net.DatagramSocket;
+import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.net.SocketException;
import java.net.UnknownHostException;
@@ -255,6 +256,12 @@ public abstract class AbstractGangliaSink implements
MetricsSink {
protected void emitToGangliaHosts() throws IOException {
try {
for (SocketAddress socketAddress : metricsServers) {
+if (socketAddress == null || !(socketAddress instanceof
InetSocketAddress))
+ throw new IllegalArgumentException(Unsupported Address type);
+InetSocketAddress inetAddress = (InetSocketAddress)socketAddress;
+if(inetAddress.isUnresolved()) {
+ throw new UnknownHostException(Unresolved host: + inetAddress);
+}
DatagramPacket packet =
new DatagramPacket(buffer, offset, socketAddress);
datagramSocket.send(packet);
