[hadoop] branch trunk updated: YARN-9270. Minor cleanup in TestFpgaDiscoverer. Contributed by Peter Bacsko.
This is an automated email from the ASF dual-hosted git repository.
devaraj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 56f1e13 YARN-9270. Minor cleanup in TestFpgaDiscoverer. Contributed
by Peter Bacsko.
56f1e13 is described below
commit 56f1e131ecb9f53dcc2596807f8dec2d2b95bb51
Author: Devaraj K
AuthorDate: Fri Mar 29 10:58:56 2019 -0700
YARN-9270. Minor cleanup in TestFpgaDiscoverer. Contributed by Peter Bacsko.
---
.../resources/fpga/FpgaResourceHandlerImpl.java| 12 +-
.../resourceplugin/fpga/FpgaDiscoverer.java| 30 +--
.../fpga/FpgaNodeResourceUpdateHandler.java| 13 +-
.../resourceplugin/fpga/FpgaResourcePlugin.java| 12 +-
.../resources/fpga/TestFpgaResourceHandler.java| 26 ++-
.../resourceplugin/fpga/TestFpgaDiscoverer.java| 211 -
6 files changed, 124 insertions(+), 180 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java
index cd1ea13..d06b805 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceHandlerImpl.java
@@ -64,6 +64,8 @@ public class FpgaResourceHandlerImpl implements
ResourceHandler {
private final CGroupsHandler cGroupsHandler;
+ private final FpgaDiscoverer fpgaDiscoverer;
+
public static final String EXCLUDED_FPGAS_CLI_OPTION = "--excluded_fpgas";
public static final String CONTAINER_ID_CLI_OPTION = "--container_id";
private PrivilegedOperationExecutor privilegedOperationExecutor;
@@ -72,10 +74,11 @@ public class FpgaResourceHandlerImpl implements
ResourceHandler {
public FpgaResourceHandlerImpl(Context nmContext,
CGroupsHandler cGroupsHandler,
PrivilegedOperationExecutor privilegedOperationExecutor,
- AbstractFpgaVendorPlugin plugin) {
+ AbstractFpgaVendorPlugin plugin,
+ FpgaDiscoverer fpgaDiscoverer) {
this.allocator = new FpgaResourceAllocator(nmContext);
this.vendorPlugin = plugin;
-FpgaDiscoverer.getInstance().setResourceHanderPlugin(vendorPlugin);
+this.fpgaDiscoverer = fpgaDiscoverer;
this.cGroupsHandler = cGroupsHandler;
this.privilegedOperationExecutor = privilegedOperationExecutor;
}
@@ -99,8 +102,7 @@ public class FpgaResourceHandlerImpl implements
ResourceHandler {
}
LOG.info("FPGA Plugin bootstrap success.");
// Get avialable devices minor numbers from toolchain or static
configuration
-List fpgaDeviceList =
-FpgaDiscoverer.getInstance().discover();
+List fpgaDeviceList = fpgaDiscoverer.discover();
allocator.addFpgaDevices(vendorPlugin.getFpgaType(), fpgaDeviceList);
this.cGroupsHandler.initializeCGroupController(
CGroupsHandler.CGroupController.DEVICES);
@@ -183,7 +185,7 @@ public class FpgaResourceHandlerImpl implements
ResourceHandler {
" if you want YARN to program the device");
} else {
LOG.info("IP file path:" + ipFilePath);
- List allowed =
allocation.getAllowed();
+ List allowed = allocation.getAllowed();
String majorMinorNumber;
for (int i = 0; i < allowed.size(); i++) {
FpgaDevice device = allowed.get(i);
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDiscoverer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDiscoverer.java
index a049038..185effa 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDiscoverer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/resourceplugin/fpga/FpgaDiscoverer.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org
[hadoop] branch trunk updated: YARN-9269. Minor cleanup in FpgaResourceAllocator. Contributed by Peter Bacsko.
This is an automated email from the ASF dual-hosted git repository.
devaraj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a4cd75e YARN-9269. Minor cleanup in FpgaResourceAllocator.
Contributed by Peter Bacsko.
a4cd75e is described below
commit a4cd75e09c934699ec5e2fa969f1c8d0a14c1d49
Author: Devaraj K
AuthorDate: Wed Mar 27 10:08:07 2019 -0700
YARN-9269. Minor cleanup in FpgaResourceAllocator. Contributed by Peter
Bacsko.
---
.../resources/fpga/FpgaResourceAllocator.java | 104 ++---
.../resources/fpga/FpgaResourceHandlerImpl.java| 2 +-
2 files changed, 53 insertions(+), 53 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceAllocator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceAllocator.java
index b64ffd0..01036db 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceAllocator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceAllocator.java
@@ -33,11 +33,15 @@ import
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resource
import java.io.IOException;
import java.io.Serializable;
-import java.util.*;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
import static org.apache.hadoop.yarn.api.records.ResourceInformation.FPGA_URI;
-
/**
* This FPGA resource allocator tends to be used by different FPGA vendor's
plugin
* A "type" parameter is taken into consideration when allocation
@@ -50,20 +54,21 @@ public class FpgaResourceAllocator {
private List allowedFpgas = new LinkedList<>();
//key is resource type of FPGA, vendor plugin supported ID
- private LinkedHashMap> availableFpga = new
LinkedHashMap<>();
+ private Map> availableFpgas = new HashMap<>();
- //key is requestor, aka. container ID
- private LinkedHashMap> usedFpgaByRequestor = new
LinkedHashMap<>();
+ //key is the container ID
+ private Map> containerToFpgaMapping =
+ new HashMap<>();
private Context nmContext;
@VisibleForTesting
- public HashMap> getAvailableFpga() {
-return availableFpga;
+ Map> getAvailableFpga() {
+return availableFpgas;
}
@VisibleForTesting
- public List getAllowedFpga() {
+ List getAllowedFpga() {
return allowedFpgas;
}
@@ -72,25 +77,31 @@ public class FpgaResourceAllocator {
}
@VisibleForTesting
- public int getAvailableFpgaCount() {
+ int getAvailableFpgaCount() {
int count = 0;
-for (List l : availableFpga.values()) {
- count += l.size();
-}
+
+count = availableFpgas.values()
+ .stream()
+ .mapToInt(i -> i.size())
+ .sum();
+
return count;
}
@VisibleForTesting
- public HashMap> getUsedFpga() {
-return usedFpgaByRequestor;
+ Map> getUsedFpga() {
+return containerToFpgaMapping;
}
@VisibleForTesting
- public int getUsedFpgaCount() {
+ int getUsedFpgaCount() {
int count = 0;
-for (List l : usedFpgaByRequestor.values()) {
- count += l.size();
-}
+
+count = containerToFpgaMapping.values()
+.stream()
+.mapToInt(i -> i.size())
+.sum();
+
return count;
}
@@ -252,42 +263,31 @@ public class FpgaResourceAllocator {
}
}
- public synchronized void addFpga(String type, List list) {
-availableFpga.putIfAbsent(type, new LinkedList<>());
+ // called once during initialization
+ public synchronized void addFpgaDevices(String type, List list) {
+availableFpgas.putIfAbsent(type, new LinkedList<>());
+List fpgaDevices = new LinkedList<>();
+
for (FpgaDevice device : list) {
if (!allowedFpgas.contains(device)) {
-allowedFpgas.add(device);
-availableFpga.get(type).add(device);
+fpgaDevices.add(device);
+availableFpgas.get(type).add(device);
+ } else {
+LOG.warn("Duplicate device found: " + device + ". Ignored");
}
}
-LOG.info("Add a list of FPGA Devices: " + list);
+
+allowedFpgas = ImmutableList.copyOf(fpgaDevices);
+LOG.info("Added a list of FPGA Devices: " + allowedFpgas);
}
publi
[hadoop] branch trunk updated: YARN-9268. General improvements in FpgaDevice. Contributed by Peter Bacsko.
This is an automated email from the ASF dual-hosted git repository.
devaraj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new eeda689 YARN-9268. General improvements in FpgaDevice. Contributed by
Peter Bacsko.
eeda689 is described below
commit eeda6891e49bc13ae86d0193f94238b7109e291d
Author: Devaraj K
AuthorDate: Mon Mar 25 13:22:53 2019 -0700
YARN-9268. General improvements in FpgaDevice. Contributed by Peter Bacsko.
---
.../resources/fpga/FpgaResourceAllocator.java | 133 +
.../fpga/AoclDiagnosticOutputParser.java | 4 +-
.../resourceplugin/fpga/FpgaDiscoverer.java| 2 +-
.../fpga/discovery/DeviceSpecParser.java | 7 +-
.../resources/fpga/TestFpgaResourceHandler.java| 32 ++---
.../resourceplugin/fpga/TestAoclOutputParser.java | 24 +---
.../resourceplugin/fpga/TestFpgaDiscoverer.java| 17 ++-
7 files changed, 88 insertions(+), 131 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceAllocator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceAllocator.java
index e5622f9..b64ffd0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceAllocator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceAllocator.java
@@ -21,6 +21,7 @@ package
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resourc
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -51,7 +52,7 @@ public class FpgaResourceAllocator {
//key is resource type of FPGA, vendor plugin supported ID
private LinkedHashMap> availableFpga = new
LinkedHashMap<>();
- //key is requetor, aka. container ID
+ //key is requestor, aka. container ID
private LinkedHashMap> usedFpgaByRequestor = new
LinkedHashMap<>();
private Context nmContext;
@@ -133,35 +134,33 @@ public class FpgaResourceAllocator {
}
}
- public static class FpgaDevice implements Comparable,
Serializable {
+ /** A class that represents an FPGA card. */
+ public static class FpgaDevice implements Serializable {
+private static final long serialVersionUID = -4678487141824092751L;
+private final String type;
+private final int major;
+private final int minor;
-private static final long serialVersionUID = 1L;
+// the alias device name. Intel use acl number acl0 to acl31
+private final String aliasDevName;
-private String type;
-private Integer major;
-private Integer minor;
-// IP file identifier. matrix multiplication for instance
+// IP file identifier. matrix multiplication for instance (mutable)
private String IPID;
-// SHA-256 hash of the uploaded aocx file
+// SHA-256 hash of the uploaded aocx file (mutable)
private String aocxHash;
-// the device name under /dev
-private String devName;
-// the alias device name. Intel use acl number acl0 to acl31
-private String aliasDevName;
-// lspci output's bus number: 02:00.00 (bus:slot.func)
-private String busNum;
-private String temperature;
-private String cardPowerUsage;
+
+// cached hash value
+private Integer hashCode;
public String getType() {
return type;
}
-public Integer getMajor() {
+public int getMajor() {
return major;
}
-public Integer getMinor() {
+public int getMinor() {
return minor;
}
@@ -181,57 +180,16 @@ public class FpgaResourceAllocator {
this.IPID = IPID;
}
-public String getDevName() {
- return devName;
-}
-
-public void setDevName(String devName) {
- this.devName = devName;
-}
-
public String getAliasDevName() {
return aliasDevName;
}
-public void setAliasDevName(String aliasDevName) {
- this.aliasDevName = aliasDevName;
-}
-
-public String getBusNum() {
- return busNum;
-}
-
-public void setBusNum(String busNum) {
- this.busNum = busNum;
-}
-
-public String getTemperature() {
- return temperature;
-}
-
-public String getCardPowerUsage() {
- return cardPowerUsage;
-}
-
-public FpgaDevice(String type, Inte
[hadoop] branch trunk updated: YARN-9267. General improvements in FpgaResourceHandlerImpl. Contributed by Peter Bacsko.
This is an automated email from the ASF dual-hosted git repository.
devaraj pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new a99eb80 YARN-9267. General improvements in FpgaResourceHandlerImpl.
Contributed by Peter Bacsko.
a99eb80 is described below
commit a99eb80659835107f4015c859b3319bf3a70c281
Author: Devaraj K
AuthorDate: Thu Mar 21 11:15:56 2019 -0700
YARN-9267. General improvements in FpgaResourceHandlerImpl. Contributed by
Peter Bacsko.
---
.../resources/fpga/FpgaResourceAllocator.java | 28 +++--
.../resources/fpga/FpgaResourceHandlerImpl.java| 59 ++
.../resources/fpga/TestFpgaResourceHandler.java| 124 +++--
3 files changed, 151 insertions(+), 60 deletions(-)
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceAllocator.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceAllocator.java
index 334c6bd..e5622f9 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceAllocator.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/fpga/FpgaResourceAllocator.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
-import
org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.fpga.FpgaDiscoverer;
import java.io.IOException;
import java.io.Serializable;
@@ -143,6 +142,8 @@ public class FpgaResourceAllocator {
private Integer minor;
// IP file identifier. matrix multiplication for instance
private String IPID;
+// SHA-256 hash of the uploaded aocx file
+private String aocxHash;
// the device name under /dev
private String devName;
// the alias device name. Intel use acl number acl0 to acl31
@@ -168,6 +169,14 @@ public class FpgaResourceAllocator {
return IPID;
}
+public String getAocxHash() {
+ return aocxHash;
+}
+
+public void setAocxHash(String hash) {
+ this.aocxHash = hash;
+}
+
public void setIPID(String IPID) {
this.IPID = IPID;
}
@@ -263,7 +272,8 @@ public class FpgaResourceAllocator {
@Override
public String toString() {
return "FPGA Device:(Type: " + this.type + ", Major: " +
- this.major + ", Minor: " + this.minor + ", IPID: " + this.IPID + ")";
+ this.major + ", Minor: " + this.minor + ", IPID: " +
+ this.IPID + ", Hash: " + this.aocxHash + ")";
}
}
@@ -279,11 +289,14 @@ public class FpgaResourceAllocator {
}
public synchronized void updateFpga(String requestor,
- FpgaDevice device, String newIPID) {
+ FpgaDevice device, String newIPID, String newHash) {
List usedFpgas = usedFpgaByRequestor.get(requestor);
int index = findMatchedFpga(usedFpgas, device);
if (-1 != index) {
usedFpgas.get(index).setIPID(newIPID);
+ FpgaDevice fpga = usedFpgas.get(index);
+ fpga.setIPID(newIPID);
+ fpga.setAocxHash(newHash);
} else {
LOG.warn("Failed to update FPGA due to unknown reason " +
"that no record for this allocated device:" + device);
@@ -307,12 +320,12 @@ public class FpgaResourceAllocator {
* @param type vendor plugin supported FPGA device type
* @param count requested FPGA slot count
* @param container container id
- * @param IPIDPreference allocate slot with this IPID first
+ * @param ipidHash hash of the localized aocx file
* @return Instance consists two List of allowed and denied {@link
FpgaDevice}
* @throws ResourceHandlerException When failed to allocate or write state
store
* */
public synchronized FpgaAllocation assignFpga(String type, long count,
- Container container, String IPIDPreference) throws
ResourceHandlerException {
+ Container container, String ipidHash) throws ResourceHandlerException {
List currentAvailableFpga = availableFpga.get(type);
String requestor = container.getContainerId().toString();
if (null == currentAvailableFpga) {
@@ -327,8 +340,9 @@ pu
hadoop git commit: MAPREDUCE-6785. ContainerLauncherImpl support for reusing the containers. Contributed by Naganarasimha G R.
Repository: hadoop
Updated Branches:
refs/heads/MR-6749 48a24ec9e -> 6507db0bc
MAPREDUCE-6785. ContainerLauncherImpl support for reusing the containers.
Contributed by Naganarasimha G R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6507db0b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6507db0b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6507db0b
Branch: refs/heads/MR-6749
Commit: 6507db0bcd9bf08e20103305f0b3acf302436fa9
Parents: 48a24ec
Author: Devaraj K <deva...@apache.org>
Authored: Tue Apr 4 15:48:35 2017 -0700
Committer: Devaraj K <deva...@apache.org>
Committed: Tue Apr 4 15:48:35 2017 -0700
--
.../TaskAttemptContainerAssignedEvent.java | 9 ++
.../v2/app/job/impl/TaskAttemptImpl.java| 98 +++-
.../mapreduce/v2/app/rm/ContainerRequestor.java | 6 ++
.../v2/app/rm/RMContainerAllocator.java | 10 +-
.../v2/app/rm/RMContainerRequestor.java | 30 +-
.../v2/app/rm/RMContainerReuseRequestor.java| 62 ++---
.../app/rm/TestRMContainerReuseRequestor.java | 30 --
7 files changed, 169 insertions(+), 76 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6507db0b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java
index 0f69fa8..41dfb03 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptContainerAssignedEvent.java
@@ -28,6 +28,7 @@ public class TaskAttemptContainerAssignedEvent extends
TaskAttemptEvent {
private final Container container;
private final Map<ApplicationAccessType, String> applicationACLs;
+ private int shufflePort = -1;
public TaskAttemptContainerAssignedEvent(TaskAttemptId id,
Container container, Map<ApplicationAccessType, String> applicationACLs)
{
@@ -36,6 +37,14 @@ public class TaskAttemptContainerAssignedEvent extends
TaskAttemptEvent {
this.applicationACLs = applicationACLs;
}
+ public int getShufflePort() {
+return shufflePort;
+ }
+
+ public void setShufflePort(int shufflePort) {
+this.shufflePort = shufflePort;
+ }
+
public Container getContainer() {
return this.container;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6507db0b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 4305824..6d3caae 100755
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -250,7 +250,8 @@ public abstract class TaskAttemptImpl implements
// Transitions from the UNASSIGNED state.
.addTransition(TaskAttemptStateInternal.UNASSIGNED,
- TaskAttemptStateInternal.ASSIGNED, TaskAttemptEventType.TA_ASSIGNED,
+ EnumSet.of(TaskAttemptStateInternal.ASSIGNED,
+ TaskAttemptStateInternal.RUNNING), TaskAttemptEventType.TA_ASSIGNED,
new ContainerAssignedTransition())
.addTransition(TaskAttemptStateInternal.UNASSIGNED,
TaskAttemptStateInternal.KILLED,
TaskAttemptEventType.TA_KILL, new DeallocateContainerTransition(
@@ -1676,13 +1677,14 @@ public abstract class TaskAttemptImpl implements
}
private static class ContainerAssignedTransition implements
- SingleArcTransition<TaskAttemptImpl, TaskAttemptEvent> {
+ Mult
[hadoop] Git Push Summary
Repository: hadoop Updated Branches: refs/heads/MR-6749 [created] ed9fcbec5 - To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
svn commit: r1739695 - /hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml
Author: devaraj Date: Mon Apr 18 06:21:54 2016 New Revision: 1739695 URL: http://svn.apache.org/viewvc?rev=1739695=rev Log: Updated my name in PMC list Modified: hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml Modified: hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml URL: http://svn.apache.org/viewvc/hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml?rev=1739695=1739694=1739695=diff == --- hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml (original) +++ hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml Mon Apr 18 06:21:54 2016 @@ -160,6 +160,14 @@ + devaraj + Devaraj K + Intel + + +5.5 + + + dhruba http://people.apache.org/~dhruba;>Dhruba Borthakur Facebook
hadoop git commit: YARN-4667. RM Admin CLI for refreshNodesResources throws NPE when nothing is configured. Contributed by Naganarasimha G R.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 5690e1120 -> 35651bab9
YARN-4667. RM Admin CLI for refreshNodesResources throws NPE when nothing
is configured. Contributed by Naganarasimha G R.
(cherry picked from commit 565af873d5e9bc4178f1221805765a417f065192)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/35651bab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/35651bab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/35651bab
Branch: refs/heads/branch-2.8
Commit: 35651bab990a6f2d37cef2afde8e15277c48bec4
Parents: 5690e11
Author: Devaraj K <deva...@apache.org>
Authored: Mon Feb 8 15:01:54 2016 +0530
Committer: Devaraj K <deva...@apache.org>
Committed: Mon Feb 8 15:04:15 2016 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../server/resourcemanager/AdminService.java| 6 ++--
.../resourcemanager/TestRMAdminService.java | 35 +---
3 files changed, 36 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35651bab/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 656146f..2f1879b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1194,6 +1194,9 @@ Release 2.8.0 - UNRELEASED
policy instead of using same as active applications ordering policy.
(Rohith Sharma K S via jianhe)
+YARN-4667. RM Admin CLI for refreshNodesResources throws NPE when nothing
+is configured. (Naganarasimha G R via devaraj)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35651bab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 353e72d..80ff3cc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -69,11 +69,11 @@ import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRespons
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest;
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest;
+import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest;
+import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesResponse;
-import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest;
-import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse;
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest;
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsResponse;
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest;
@@ -635,7 +635,7 @@ public class AdminService extends CompositeService
implements
newconf = new DynamicResourceConfiguration(configuration, true);
}
- if (newconf.getNodes().length == 0) {
+ if (newconf.getNodes() == null || newconf.getNodes().length == 0) {
RMAuditLogger.logSuccess(user.getShortUserName(), argName,
"AdminService");
return response;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/35651bab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/ja
hadoop git commit: YARN-4667. RM Admin CLI for refreshNodesResources throws NPE when nothing is configured. Contributed by Naganarasimha G R.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 fd971debe -> 329df98f4
YARN-4667. RM Admin CLI for refreshNodesResources throws NPE when nothing
is configured. Contributed by Naganarasimha G R.
(cherry picked from commit 565af873d5e9bc4178f1221805765a417f065192)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/329df98f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/329df98f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/329df98f
Branch: refs/heads/branch-2
Commit: 329df98f49fd4991711ccb0789eebfb20cf86bd4
Parents: fd971de
Author: Devaraj K <deva...@apache.org>
Authored: Mon Feb 8 15:01:54 2016 +0530
Committer: Devaraj K <deva...@apache.org>
Committed: Mon Feb 8 15:03:11 2016 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../server/resourcemanager/AdminService.java| 6 ++--
.../resourcemanager/TestRMAdminService.java | 35 +---
3 files changed, 36 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/329df98f/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7859fea..634136e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1336,6 +1336,9 @@ Release 2.8.0 - UNRELEASED
policy instead of using same as active applications ordering policy.
(Rohith Sharma K S via jianhe)
+YARN-4667. RM Admin CLI for refreshNodesResources throws NPE when nothing
+is configured. (Naganarasimha G R via devaraj)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/329df98f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index fcce722..fbc2d6f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -69,11 +69,11 @@ import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRespons
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest;
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest;
+import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest;
+import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesResponse;
-import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest;
-import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse;
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest;
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsResponse;
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest;
@@ -652,7 +652,7 @@ public class AdminService extends CompositeService
implements
newconf = new DynamicResourceConfiguration(configuration, true);
}
- if (newconf.getNodes().length == 0) {
+ if (newconf.getNodes() == null || newconf.getNodes().length == 0) {
RMAuditLogger.logSuccess(user.getShortUserName(), argName,
"AdminService");
return response;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/329df98f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/
hadoop git commit: YARN-4667. RM Admin CLI for refreshNodesResources throws NPE when nothing is configured. Contributed by Naganarasimha G R.
Repository: hadoop
Updated Branches:
refs/heads/trunk f3bbe0bd0 -> 565af873d
YARN-4667. RM Admin CLI for refreshNodesResources throws NPE when nothing
is configured. Contributed by Naganarasimha G R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/565af873
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/565af873
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/565af873
Branch: refs/heads/trunk
Commit: 565af873d5e9bc4178f1221805765a417f065192
Parents: f3bbe0b
Author: Devaraj K <deva...@apache.org>
Authored: Mon Feb 8 15:01:54 2016 +0530
Committer: Devaraj K <deva...@apache.org>
Committed: Mon Feb 8 15:01:54 2016 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../server/resourcemanager/AdminService.java| 6 ++--
.../resourcemanager/TestRMAdminService.java | 35 +---
3 files changed, 36 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/565af873/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e551a2a..7ac2b39 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1391,6 +1391,9 @@ Release 2.8.0 - UNRELEASED
policy instead of using same as active applications ordering policy.
(Rohith Sharma K S via jianhe)
+YARN-4667. RM Admin CLI for refreshNodesResources throws NPE when nothing
+is configured. (Naganarasimha G R via devaraj)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/565af873/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index fcce722..fbc2d6f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -69,11 +69,11 @@ import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRespons
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest;
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest;
+import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest;
+import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesResponse;
-import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest;
-import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse;
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsRequest;
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshServiceAclsResponse;
import
org.apache.hadoop.yarn.server.api.protocolrecords.RefreshSuperUserGroupsConfigurationRequest;
@@ -652,7 +652,7 @@ public class AdminService extends CompositeService
implements
newconf = new DynamicResourceConfiguration(configuration, true);
}
- if (newconf.getNodes().length == 0) {
+ if (newconf.getNodes() == null || newconf.getNodes().length == 0) {
RMAuditLogger.logSuccess(user.getShortUserName(), argName,
"AdminService");
return response;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/565af873/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
b/hadoop
hadoop git commit: YARN-4100. Add Documentation for Distributed and Delegated-Centralized Node Labels feature. Contributed by Naganarasimha G R.
Repository: hadoop Updated Branches: refs/heads/branch-2.8 c487453b9 -> aeea77ce1 YARN-4100. Add Documentation for Distributed and Delegated-Centralized Node Labels feature. Contributed by Naganarasimha G R. (cherry picked from commit db144eb1c51c1f37bdd1e0c18e9a5b0969c82e33) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aeea77ce Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aeea77ce Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aeea77ce Branch: refs/heads/branch-2.8 Commit: aeea77ce147c8f53a868274654df693437e1c435 Parents: c487453 Author: Devaraj K <deva...@apache.org> Authored: Tue Feb 2 12:06:51 2016 +0530 Committer: Devaraj K <deva...@apache.org> Committed: Tue Feb 2 12:08:56 2016 +0530 -- hadoop-yarn-project/CHANGES.txt | 3 + .../src/main/resources/yarn-default.xml | 50 ++-- .../src/site/markdown/NodeLabel.md | 86 3 files changed, 99 insertions(+), 40 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/aeea77ce/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 901a1eb..636db91 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -592,6 +592,9 @@ Release 2.8.0 - UNRELEASED YARN-4340. Add "list" API to reservation system. (Sean Po via wangda) +YARN-4100. Add Documentation for Distributed and Delegated-Centralized +Node Labels feature. (Naganarasimha G R via devaraj) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/aeea77ce/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0add988..80f0fea 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2281,26 +2281,26 @@ -When "yarn.node-labels.configuration-type" parameter in RM is configured as -"distributed", Administrators can configure in NM, the provider for the +When "yarn.node-labels.configuration-type" is configured with "distributed" +in RM, Administrators can configure in NM the provider for the node labels by configuring this parameter. Administrators can -specify "config", "script" or the class name of the provider. Configured +configure "config", "script" or the class name of the provider. Configured class needs to extend org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider. -If "config" is specified then "ConfigurationNodeLabelsProvider" and -"script" then "ScriptNodeLabelsProvider" will be used. +If "config" is configured, then "ConfigurationNodeLabelsProvider" and if +"script" is configured, then "ScriptNodeLabelsProvider" will be used. yarn.nodemanager.node-labels.provider -When node labels "yarn.nodemanager.node-labels.provider" is of type -"config" or the configured class extends AbstractNodeLabelsProvider then -periodically node labels are retrieved from the node labels provider. -This configuration is to define the interval. If -1 is configured then -node labels are retrieved from. provider only during initialization. -Defaults to 10 mins. +When "yarn.nodemanager.node-labels.provider" is configured with "config", +"Script" or the configured class extends AbstractNodeLabelsProvider, then +periodically node labels are retrieved from the node labels provider. This +configuration is to define the interval period. +If -1 is configured then node labels are retrieved from provider only +during initialization. Defaults to 10 mins. yarn.nodemanager.node-labels.provider.fetch-interval-ms 60 @@ -2308,8 +2308,8 @@ - Interval at which node labels syncs with RM from NM.Will send loaded labels - every x intervals configured along with heartbeat from NM to RM. + Interval at which NM syncs its node labels with RM. NM will send its loaded + labels every x interva
hadoop git commit: YARN-4100. Add Documentation for Distributed and Delegated-Centralized Node Labels feature. Contributed by Naganarasimha G R.
Repository: hadoop Updated Branches: refs/heads/trunk 1cd55e0c1 -> db144eb1c YARN-4100. Add Documentation for Distributed and Delegated-Centralized Node Labels feature. Contributed by Naganarasimha G R. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db144eb1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db144eb1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db144eb1 Branch: refs/heads/trunk Commit: db144eb1c51c1f37bdd1e0c18e9a5b0969c82e33 Parents: 1cd55e0 Author: Devaraj K <deva...@apache.org> Authored: Tue Feb 2 12:06:51 2016 +0530 Committer: Devaraj K <deva...@apache.org> Committed: Tue Feb 2 12:06:51 2016 +0530 -- hadoop-yarn-project/CHANGES.txt | 3 + .../src/main/resources/yarn-default.xml | 50 ++-- .../src/site/markdown/NodeLabel.md | 86 3 files changed, 99 insertions(+), 40 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/db144eb1/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index bf46864..345c64b 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -778,6 +778,9 @@ Release 2.8.0 - UNRELEASED YARN-4340. Add "list" API to reservation system. (Sean Po via wangda) +YARN-4100. Add Documentation for Distributed and Delegated-Centralized +Node Labels feature. (Naganarasimha G R via devaraj) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/db144eb1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index e33d23e..d8ea3ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2281,26 +2281,26 @@ -When "yarn.node-labels.configuration-type" parameter in RM is configured as -"distributed", Administrators can configure in NM, the provider for the +When "yarn.node-labels.configuration-type" is configured with "distributed" +in RM, Administrators can configure in NM the provider for the node labels by configuring this parameter. Administrators can -specify "config", "script" or the class name of the provider. Configured +configure "config", "script" or the class name of the provider. Configured class needs to extend org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider. -If "config" is specified then "ConfigurationNodeLabelsProvider" and -"script" then "ScriptNodeLabelsProvider" will be used. +If "config" is configured, then "ConfigurationNodeLabelsProvider" and if +"script" is configured, then "ScriptNodeLabelsProvider" will be used. yarn.nodemanager.node-labels.provider -When node labels "yarn.nodemanager.node-labels.provider" is of type -"config" or the configured class extends AbstractNodeLabelsProvider then -periodically node labels are retrieved from the node labels provider. -This configuration is to define the interval. If -1 is configured then -node labels are retrieved from. provider only during initialization. -Defaults to 10 mins. +When "yarn.nodemanager.node-labels.provider" is configured with "config", +"Script" or the configured class extends AbstractNodeLabelsProvider, then +periodically node labels are retrieved from the node labels provider. This +configuration is to define the interval period. +If -1 is configured then node labels are retrieved from provider only +during initialization. Defaults to 10 mins. yarn.nodemanager.node-labels.provider.fetch-interval-ms 60 @@ -2308,8 +2308,8 @@ - Interval at which node labels syncs with RM from NM.Will send loaded labels - every x intervals configured along with heartbeat from NM to RM. + Interval at which NM syncs its node labels with RM. NM will send its loaded + labels every x intervals configured, along with heartbeat to RM. yarn.nodemanager.node-labels.re
hadoop git commit: YARN-4100. Add Documentation for Distributed and Delegated-Centralized Node Labels feature. Contributed by Naganarasimha G R.
Repository: hadoop Updated Branches: refs/heads/branch-2 bbda44688 -> 43131af1e YARN-4100. Add Documentation for Distributed and Delegated-Centralized Node Labels feature. Contributed by Naganarasimha G R. (cherry picked from commit db144eb1c51c1f37bdd1e0c18e9a5b0969c82e33) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43131af1 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43131af1 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43131af1 Branch: refs/heads/branch-2 Commit: 43131af1e21cde903c0203e935545251a32136f1 Parents: bbda446 Author: Devaraj K <deva...@apache.org> Authored: Tue Feb 2 12:06:51 2016 +0530 Committer: Devaraj K <deva...@apache.org> Committed: Tue Feb 2 12:07:49 2016 +0530 -- hadoop-yarn-project/CHANGES.txt | 3 + .../src/main/resources/yarn-default.xml | 50 ++-- .../src/site/markdown/NodeLabel.md | 86 3 files changed, 99 insertions(+), 40 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/43131af1/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 94b62e1..d6e1d4d 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -723,6 +723,9 @@ Release 2.8.0 - UNRELEASED YARN-4340. Add "list" API to reservation system. (Sean Po via wangda) +YARN-4100. Add Documentation for Distributed and Delegated-Centralized +Node Labels feature. (Naganarasimha G R via devaraj) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/43131af1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index e33d23e..d8ea3ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -2281,26 +2281,26 @@ -When "yarn.node-labels.configuration-type" parameter in RM is configured as -"distributed", Administrators can configure in NM, the provider for the +When "yarn.node-labels.configuration-type" is configured with "distributed" +in RM, Administrators can configure in NM the provider for the node labels by configuring this parameter. Administrators can -specify "config", "script" or the class name of the provider. Configured +configure "config", "script" or the class name of the provider. Configured class needs to extend org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider. -If "config" is specified then "ConfigurationNodeLabelsProvider" and -"script" then "ScriptNodeLabelsProvider" will be used. +If "config" is configured, then "ConfigurationNodeLabelsProvider" and if +"script" is configured, then "ScriptNodeLabelsProvider" will be used. yarn.nodemanager.node-labels.provider -When node labels "yarn.nodemanager.node-labels.provider" is of type -"config" or the configured class extends AbstractNodeLabelsProvider then -periodically node labels are retrieved from the node labels provider. -This configuration is to define the interval. If -1 is configured then -node labels are retrieved from. provider only during initialization. -Defaults to 10 mins. +When "yarn.nodemanager.node-labels.provider" is configured with "config", +"Script" or the configured class extends AbstractNodeLabelsProvider, then +periodically node labels are retrieved from the node labels provider. This +configuration is to define the interval period. +If -1 is configured then node labels are retrieved from provider only +during initialization. Defaults to 10 mins. yarn.nodemanager.node-labels.provider.fetch-interval-ms 60 @@ -2308,8 +2308,8 @@ - Interval at which node labels syncs with RM from NM.Will send loaded labels - every x intervals configured along with heartbeat from NM to RM. + Interval at which NM syncs its node labels with RM. NM will send its loaded + labels every x interva
hadoop git commit: YARN-4411. RMAppAttemptImpl#createApplicationAttemptReport throws IllegalArgumentException. Contributed by Bibin A Chundatt and yarntime.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 6ce9b1240 -> 123269c07
YARN-4411. RMAppAttemptImpl#createApplicationAttemptReport throws
IllegalArgumentException. Contributed by Bibin A Chundatt and yarntime.
(cherry picked from commit a277bdc9edc66bef419fcd063b832073e512f234)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/123269c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/123269c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/123269c0
Branch: refs/heads/branch-2
Commit: 123269c07f94b1792aea2613d78c019a9e53a8aa
Parents: 6ce9b12
Author: Devaraj K <deva...@apache.org>
Authored: Fri Jan 29 13:51:37 2016 +0530
Committer: Devaraj K <deva...@apache.org>
Committed: Fri Jan 29 13:52:47 2016 +0530
--
hadoop-yarn-project/CHANGES.txt| 3 +++
.../rmapp/attempt/RMAppAttemptImpl.java| 10 +-
.../rmapp/attempt/TestRMAppAttemptTransitions.java | 13 +
3 files changed, 21 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/123269c0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 78e981c..fd9cf3a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -334,6 +334,9 @@ Release 2.8.0 - UNRELEASED
YARN-4219. New levelDB cache storage for timeline v1.5. (Li Lu via xgong)
+YARN-4411. RMAppAttemptImpl#createApplicationAttemptReport throws
+IllegalArgumentException. (Bibin A Chundatt, yarntime via devaraj)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/123269c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 3f45cb4..99f1b21 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -2067,11 +2067,11 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
// am container.
ContainerId amId =
masterContainer == null ? null : masterContainer.getId();
- attemptReport = ApplicationAttemptReport.newInstance(this
- .getAppAttemptId(), this.getHost(), this.getRpcPort(), this
- .getTrackingUrl(), this.getOriginalTrackingUrl(),
this.getDiagnostics(),
- YarnApplicationAttemptState.valueOf(this.getState().toString()),
- amId, this.startTime, this.finishTime);
+ attemptReport = ApplicationAttemptReport.newInstance(
+ this.getAppAttemptId(), this.getHost(), this.getRpcPort(),
+ this.getTrackingUrl(), this.getOriginalTrackingUrl(),
+ this.getDiagnostics(), createApplicationAttemptState(), amId,
+ this.startTime, this.finishTime);
} finally {
this.readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/123269c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 7f9610f..e7985fa 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++
b/hadoop-yarn-project/hadoop-y
hadoop git commit: YARN-4411. RMAppAttemptImpl#createApplicationAttemptReport throws IllegalArgumentException. Contributed by Bibin A Chundatt and yarntime.
Repository: hadoop
Updated Branches:
refs/heads/trunk df99ea8a9 -> a277bdc9e
YARN-4411. RMAppAttemptImpl#createApplicationAttemptReport throws
IllegalArgumentException. Contributed by Bibin A Chundatt and yarntime.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a277bdc9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a277bdc9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a277bdc9
Branch: refs/heads/trunk
Commit: a277bdc9edc66bef419fcd063b832073e512f234
Parents: df99ea8
Author: Devaraj K <deva...@apache.org>
Authored: Fri Jan 29 13:51:37 2016 +0530
Committer: Devaraj K <deva...@apache.org>
Committed: Fri Jan 29 13:51:37 2016 +0530
--
hadoop-yarn-project/CHANGES.txt| 3 +++
.../rmapp/attempt/RMAppAttemptImpl.java| 10 +-
.../rmapp/attempt/TestRMAppAttemptTransitions.java | 13 +
3 files changed, 21 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a277bdc9/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index dd61f2a..138e581 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -392,6 +392,9 @@ Release 2.8.0 - UNRELEASED
YARN-4219. New levelDB cache storage for timeline v1.5. (Li Lu via xgong)
+YARN-4411. RMAppAttemptImpl#createApplicationAttemptReport throws
+IllegalArgumentException. (Bibin A Chundatt, yarntime via devaraj)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a277bdc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 3f45cb4..99f1b21 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -2067,11 +2067,11 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
// am container.
ContainerId amId =
masterContainer == null ? null : masterContainer.getId();
- attemptReport = ApplicationAttemptReport.newInstance(this
- .getAppAttemptId(), this.getHost(), this.getRpcPort(), this
- .getTrackingUrl(), this.getOriginalTrackingUrl(),
this.getDiagnostics(),
- YarnApplicationAttemptState.valueOf(this.getState().toString()),
- amId, this.startTime, this.finishTime);
+ attemptReport = ApplicationAttemptReport.newInstance(
+ this.getAppAttemptId(), this.getHost(), this.getRpcPort(),
+ this.getTrackingUrl(), this.getOriginalTrackingUrl(),
+ this.getDiagnostics(), createApplicationAttemptState(), amId,
+ this.startTime, this.finishTime);
} finally {
this.readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a277bdc9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 7f9610f..e7985fa 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apa
hadoop git commit: YARN-4411. RMAppAttemptImpl#createApplicationAttemptReport throws IllegalArgumentException. Contributed by Bibin A Chundatt and yarntime.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 ce5947916 -> eb2366eda
YARN-4411. RMAppAttemptImpl#createApplicationAttemptReport throws
IllegalArgumentException. Contributed by Bibin A Chundatt and yarntime.
(cherry picked from commit a277bdc9edc66bef419fcd063b832073e512f234)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb2366ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb2366ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb2366ed
Branch: refs/heads/branch-2.8
Commit: eb2366eda1987a2c71ed90c67d354deba57ee6fc
Parents: ce59479
Author: Devaraj K <deva...@apache.org>
Authored: Fri Jan 29 13:51:37 2016 +0530
Committer: Devaraj K <deva...@apache.org>
Committed: Fri Jan 29 13:53:29 2016 +0530
--
hadoop-yarn-project/CHANGES.txt| 3 +++
.../rmapp/attempt/RMAppAttemptImpl.java| 10 +-
.../rmapp/attempt/TestRMAppAttemptTransitions.java | 13 +
3 files changed, 21 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb2366ed/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a4a3f3f..1ba981d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -216,6 +216,9 @@ Release 2.8.0 - UNRELEASED
YARN-4219. New levelDB cache storage for timeline v1.5. (Li Lu via xgong)
+YARN-4411. RMAppAttemptImpl#createApplicationAttemptReport throws
+IllegalArgumentException. (Bibin A Chundatt, yarntime via devaraj)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb2366ed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 8aefe9f..1ff4d75 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -2067,11 +2067,11 @@ public class RMAppAttemptImpl implements RMAppAttempt,
Recoverable {
// am container.
ContainerId amId =
masterContainer == null ? null : masterContainer.getId();
- attemptReport = ApplicationAttemptReport.newInstance(this
- .getAppAttemptId(), this.getHost(), this.getRpcPort(), this
- .getTrackingUrl(), this.getOriginalTrackingUrl(),
this.getDiagnostics(),
- YarnApplicationAttemptState.valueOf(this.getState().toString()),
- amId, this.startTime, this.finishTime);
+ attemptReport = ApplicationAttemptReport.newInstance(
+ this.getAppAttemptId(), this.getHost(), this.getRpcPort(),
+ this.getTrackingUrl(), this.getOriginalTrackingUrl(),
+ this.getDiagnostics(), createApplicationAttemptState(), amId,
+ this.startTime, this.finishTime);
} finally {
this.readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb2366ed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 7f9610f..e7985fa 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++
b/hadoop-yarn-project/hadoop-y
hadoop git commit: YARN-3964. Support NodeLabelsProvider at Resource Manager side. Contributed by Dian Fu.
Repository: hadoop
Updated Branches:
refs/heads/trunk 7e2c971fe -> db9304788
YARN-3964. Support NodeLabelsProvider at Resource Manager side.
Contributed by Dian Fu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db930478
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db930478
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db930478
Branch: refs/heads/trunk
Commit: db9304788187c700647c4d84caeb3b5ad6d868d8
Parents: 7e2c971
Author: Devaraj K <deva...@apache.org>
Authored: Sun Oct 11 11:21:29 2015 +0530
Committer: Devaraj K <deva...@apache.org>
Committed: Sun Oct 11 11:21:29 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 40 +++-
.../nodelabels/CommonNodeLabelsManager.java | 18 +-
.../src/main/resources/yarn-default.xml | 28 ++-
.../server/resourcemanager/AdminService.java| 26 +--
.../resourcemanager/RMActiveServiceContext.java | 15 ++
.../yarn/server/resourcemanager/RMContext.java | 6 +
.../server/resourcemanager/RMContextImpl.java | 13 ++
.../server/resourcemanager/ResourceManager.java | 22 ++
.../resourcemanager/ResourceTrackerService.java | 25 +--
.../nodelabels/NodeLabelsUtils.java | 59 ++
.../RMDelegatedNodeLabelsUpdater.java | 211 +++
.../nodelabels/RMNodeLabelsMappingProvider.java | 45
.../resourcemanager/webapp/RMWebServices.java | 21 +-
.../resourcemanager/TestRMAdminService.java | 8 +-
.../TestResourceTrackerService.java | 5 +-
.../TestRMDelegatedNodeLabelsUpdater.java | 163 ++
.../webapp/TestRMWebServicesNodeLabels.java | 4 +-
18 files changed, 648 insertions(+), 64 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db930478/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index de8f678..3762e45 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -229,6 +229,9 @@ Release 2.8.0 - UNRELEASED
YARN-261. Ability to fail AM attempts (Andrey Klochkov and
Rohith Sharma K S via jlowe)
+YARN-3964. Support NodeLabelsProvider at Resource Manager side.
+ (Dian Fu via devaraj)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db930478/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 1e102e5..8d34f4e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1999,14 +1999,17 @@ public class YarnConfiguration extends Configuration {
public static final String NODELABEL_CONFIGURATION_TYPE =
NODE_LABELS_PREFIX + "configuration-type";
- public static final String CENTALIZED_NODELABEL_CONFIGURATION_TYPE =
+ public static final String CENTRALIZED_NODELABEL_CONFIGURATION_TYPE =
"centralized";
-
+
+ public static final String DELEGATED_CENTALIZED_NODELABEL_CONFIGURATION_TYPE
=
+ "delegated-centralized";
+
public static final String DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE =
"distributed";
public static final String DEFAULT_NODELABEL_CONFIGURATION_TYPE =
- CENTALIZED_NODELABEL_CONFIGURATION_TYPE;
+ CENTRALIZED_NODELABEL_CONFIGURATION_TYPE;
public static final String MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY =
YARN_PREFIX + "cluster.max-application-priority";
@@ -2019,6 +2022,20 @@ public class YarnConfiguration extends Configuration {
NODELABEL_CONFIGURATION_TYPE, DEFAULT_NODELABEL_CONFIGURATION_TYPE));
}
+ @Private
+ public static boolean isCentralizedNodeLabelConfiguration(
+ Configuration conf) {
+return CENTRALIZED_NODELABEL_CONFIGURATION_TYPE.equals(conf.get(
+NODELABEL_CONFIGURATION_TYPE, DEFAULT_NODELABEL_CONFIGURATION_TYPE));
+ }
+
+ @Private
+ public static boolean isDelegatedCentralizedNodeLabelConfiguration(
+ Configuration conf) {
+return DELEGATED_CENTALIZED_NODELABEL_CONFIGURATION_TYPE.equals(conf.get(
+N
hadoop git commit: YARN-3964. Support NodeLabelsProvider at Resource Manager side. Contributed by Dian Fu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 f6b0fcdc7 -> 28edc7b12
YARN-3964. Support NodeLabelsProvider at Resource Manager side.
Contributed by Dian Fu.
(cherry picked from commit db9304788187c700647c4d84caeb3b5ad6d868d8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28edc7b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28edc7b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28edc7b1
Branch: refs/heads/branch-2
Commit: 28edc7b1296c2cead1c290c377db2881b9dbb9fd
Parents: f6b0fcd
Author: Devaraj K <deva...@apache.org>
Authored: Sun Oct 11 11:21:29 2015 +0530
Committer: Devaraj K <deva...@apache.org>
Committed: Sun Oct 11 11:24:05 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 40 +++-
.../nodelabels/CommonNodeLabelsManager.java | 18 +-
.../src/main/resources/yarn-default.xml | 28 ++-
.../server/resourcemanager/AdminService.java| 26 +--
.../resourcemanager/RMActiveServiceContext.java | 15 ++
.../yarn/server/resourcemanager/RMContext.java | 6 +
.../server/resourcemanager/RMContextImpl.java | 13 ++
.../server/resourcemanager/ResourceManager.java | 22 ++
.../resourcemanager/ResourceTrackerService.java | 25 +--
.../nodelabels/NodeLabelsUtils.java | 59 ++
.../RMDelegatedNodeLabelsUpdater.java | 211 +++
.../nodelabels/RMNodeLabelsMappingProvider.java | 45
.../resourcemanager/webapp/RMWebServices.java | 21 +-
.../resourcemanager/TestRMAdminService.java | 8 +-
.../TestResourceTrackerService.java | 5 +-
.../TestRMDelegatedNodeLabelsUpdater.java | 163 ++
.../webapp/TestRMWebServicesNodeLabels.java | 4 +-
18 files changed, 648 insertions(+), 64 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28edc7b1/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f278b98..8ef524f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -174,6 +174,9 @@ Release 2.8.0 - UNRELEASED
YARN-261. Ability to fail AM attempts (Andrey Klochkov and
Rohith Sharma K S via jlowe)
+YARN-3964. Support NodeLabelsProvider at Resource Manager side.
+ (Dian Fu via devaraj)
+
IMPROVEMENTS
YARN-644. Basic null check is not performed on passed in arguments before
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28edc7b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 654a493..3af0c62 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1999,14 +1999,17 @@ public class YarnConfiguration extends Configuration {
public static final String NODELABEL_CONFIGURATION_TYPE =
NODE_LABELS_PREFIX + "configuration-type";
- public static final String CENTALIZED_NODELABEL_CONFIGURATION_TYPE =
+ public static final String CENTRALIZED_NODELABEL_CONFIGURATION_TYPE =
"centralized";
-
+
+ public static final String DELEGATED_CENTALIZED_NODELABEL_CONFIGURATION_TYPE
=
+ "delegated-centralized";
+
public static final String DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE =
"distributed";
public static final String DEFAULT_NODELABEL_CONFIGURATION_TYPE =
- CENTALIZED_NODELABEL_CONFIGURATION_TYPE;
+ CENTRALIZED_NODELABEL_CONFIGURATION_TYPE;
public static final String MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY =
YARN_PREFIX + "cluster.max-application-priority";
@@ -2019,6 +2022,20 @@ public class YarnConfiguration extends Configuration {
NODELABEL_CONFIGURATION_TYPE, DEFAULT_NODELABEL_CONFIGURATION_TYPE));
}
+ @Private
+ public static boolean isCentralizedNodeLabelConfiguration(
+ Configuration conf) {
+return CENTRALIZED_NODELABEL_CONFIGURATION_TYPE.equals(conf.get(
+NODELABEL_CONFIGURATION_TYPE, DEFAULT_NODELABEL_CONFIGURATION_TYPE));
+ }
+
+ @Private
+ public static boolean isDelegatedCentralizedNodeLabelConfiguration(
+ Configuration conf) {
+return
hadoop git commit: MAPREDUCE-6426. TestShuffleHandler#testGetMapOutputInfo is failing. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d17a7bbf6 - 3074d7082
MAPREDUCE-6426. TestShuffleHandler#testGetMapOutputInfo is failing.
Contributed by zhihai xu.
(cherry picked from commit fffb15bb431fcdd7ca4ce0e249f9d45f4968497b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3074d708
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3074d708
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3074d708
Branch: refs/heads/branch-2
Commit: 3074d7082e779c1f32c60401887186776a61bdaf
Parents: d17a7bb
Author: Devaraj K deva...@apache.org
Authored: Thu Jul 9 15:00:26 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Jul 9 15:02:09 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../test/java/org/apache/hadoop/mapred/TestShuffleHandler.java | 6 +-
2 files changed, 8 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3074d708/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index bfd5b75..154a98f 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -281,6 +281,9 @@ Release 2.7.2 - UNRELEASED
MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo
if mapId is not in the cache. (zhihai xu via devaraj)
+MAPREDUCE-6426. TestShuffleHandler#testGetMapOutputInfo is failing.
+(zhihai xu via devaraj)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3074d708/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 35d9bdd..878bc87 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -837,6 +837,9 @@ public class TestShuffleHandler {
Configuration conf = new Configuration();
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+simple);
+UserGroupInformation.setConfiguration(conf);
File absLogDir = new File(target, TestShuffleHandler.class.
getSimpleName() + LocDir).getAbsoluteFile();
conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
@@ -924,7 +927,8 @@ public class TestShuffleHandler {
} catch (EOFException e) {
// ignore
}
- Assert.assertEquals(failures.size(), 0);
+ Assert.assertEquals(sendError called due to shuffle error,
+ 0, failures.size());
} finally {
shuffleHandler.stop();
FileUtil.fullyDelete(absLogDir);
hadoop git commit: MAPREDUCE-6426. TestShuffleHandler#testGetMapOutputInfo is failing. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/trunk 63d036508 - fffb15bb4
MAPREDUCE-6426. TestShuffleHandler#testGetMapOutputInfo is failing.
Contributed by zhihai xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fffb15bb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fffb15bb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fffb15bb
Branch: refs/heads/trunk
Commit: fffb15bb431fcdd7ca4ce0e249f9d45f4968497b
Parents: 63d0365
Author: Devaraj K deva...@apache.org
Authored: Thu Jul 9 15:00:26 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Jul 9 15:00:26 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../test/java/org/apache/hadoop/mapred/TestShuffleHandler.java | 6 +-
2 files changed, 8 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fffb15bb/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 874ecea..81b202b 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -548,6 +548,9 @@ Release 2.7.2 - UNRELEASED
MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo
if mapId is not in the cache. (zhihai xu via devaraj)
+MAPREDUCE-6426. TestShuffleHandler#testGetMapOutputInfo is failing.
+(zhihai xu via devaraj)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fffb15bb/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 746071f..bad9b2d 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -837,6 +837,9 @@ public class TestShuffleHandler {
Configuration conf = new Configuration();
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+simple);
+UserGroupInformation.setConfiguration(conf);
File absLogDir = new File(target, TestShuffleHandler.class.
getSimpleName() + LocDir).getAbsoluteFile();
conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
@@ -924,7 +927,8 @@ public class TestShuffleHandler {
} catch (EOFException e) {
// ignore
}
- Assert.assertEquals(failures.size(), 0);
+ Assert.assertEquals(sendError called due to shuffle error,
+ 0, failures.size());
} finally {
shuffleHandler.stop();
FileUtil.fullyDelete(absLogDir);
hadoop git commit: MAPREDUCE-6426. TestShuffleHandler#testGetMapOutputInfo is failing. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 148b4db7e - 7e5f73a95
MAPREDUCE-6426. TestShuffleHandler#testGetMapOutputInfo is failing.
Contributed by zhihai xu.
(cherry picked from commit fffb15bb431fcdd7ca4ce0e249f9d45f4968497b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e5f73a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e5f73a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e5f73a9
Branch: refs/heads/branch-2.7
Commit: 7e5f73a95274cc2d1ec502591730d90767e62da3
Parents: 148b4db
Author: Devaraj K deva...@apache.org
Authored: Thu Jul 9 15:00:26 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Jul 9 15:05:11 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../test/java/org/apache/hadoop/mapred/TestShuffleHandler.java | 6 +-
2 files changed, 8 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e5f73a9/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 93dab77..9b72dee 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -15,6 +15,9 @@ Release 2.7.2 - UNRELEASED
MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo
if mapId is not in the cache. (zhihai xu via devaraj)
+MAPREDUCE-6426. TestShuffleHandler#testGetMapOutputInfo is failing.
+(zhihai xu via devaraj)
+
Release 2.7.1 - 2015-07-06
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e5f73a9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 35d9bdd..878bc87 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -837,6 +837,9 @@ public class TestShuffleHandler {
Configuration conf = new Configuration();
conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
+simple);
+UserGroupInformation.setConfiguration(conf);
File absLogDir = new File(target, TestShuffleHandler.class.
getSimpleName() + LocDir).getAbsoluteFile();
conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
@@ -924,7 +927,8 @@ public class TestShuffleHandler {
} catch (EOFException e) {
// ignore
}
- Assert.assertEquals(failures.size(), 0);
+ Assert.assertEquals(sendError called due to shuffle error,
+ 0, failures.size());
} finally {
shuffleHandler.stop();
FileUtil.fullyDelete(absLogDir);
hadoop git commit: YARN-3837. javadocs of TimelineAuthenticationFilterInitializer give wrong prefix for auth options. Contributed by Bibin A Chundatt.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 a8bf34cad - f558429ef
YARN-3837. javadocs of TimelineAuthenticationFilterInitializer give wrong
prefix for auth options. Contributed by Bibin A Chundatt.
(cherry picked from commit af63427c6d7d2fc251eafb1f152b7a90c5bd07e5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f558429e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f558429e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f558429e
Branch: refs/heads/branch-2
Commit: f558429efef4fa1a680442dae7e8ab46b617912c
Parents: a8bf34c
Author: Devaraj K deva...@apache.org
Authored: Tue Jul 7 12:06:30 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Tue Jul 7 12:08:03 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../security/TimelineAuthenticationFilterInitializer.java | 5 ++---
2 files changed, 5 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f558429e/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6badc0d..74ea705 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -544,6 +544,9 @@ Release 2.8.0 - UNRELEASED
YARN-3882. AggregatedLogFormat should close aclScanner and ownerScanner
after create them. (zhihai xu via xgong)
+YARN-3837. javadocs of TimelineAuthenticationFilterInitializer give wrong
+prefix for auth options. (Bibin A Chundatt via devaraj)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f558429e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
index a3c136c..4e7c29a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
@@ -62,9 +62,8 @@ public class TimelineAuthenticationFilterInitializer extends
FilterInitializer {
* Initializes {@link TimelineAuthenticationFilter}
* p
* Propagates to {@link TimelineAuthenticationFilter} configuration all YARN
- * configuration properties prefixed with
- * {@code yarn.timeline-service.authentication.}
- *
+ * configuration properties prefixed with {@value #PREFIX}
+ *
* @param container
* The filter container
* @param conf
hadoop git commit: YARN-3837. javadocs of TimelineAuthenticationFilterInitializer give wrong prefix for auth options. Contributed by Bibin A Chundatt.
Repository: hadoop
Updated Branches:
refs/heads/trunk 81f364437 - af63427c6
YARN-3837. javadocs of TimelineAuthenticationFilterInitializer give wrong
prefix for auth options. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af63427c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af63427c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af63427c
Branch: refs/heads/trunk
Commit: af63427c6d7d2fc251eafb1f152b7a90c5bd07e5
Parents: 81f3644
Author: Devaraj K deva...@apache.org
Authored: Tue Jul 7 12:06:30 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Tue Jul 7 12:06:30 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../security/TimelineAuthenticationFilterInitializer.java | 5 ++---
2 files changed, 5 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af63427c/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f35be75..8f63187 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -592,6 +592,9 @@ Release 2.8.0 - UNRELEASED
YARN-3882. AggregatedLogFormat should close aclScanner and ownerScanner
after create them. (zhihai xu via xgong)
+YARN-3837. javadocs of TimelineAuthenticationFilterInitializer give wrong
+prefix for auth options. (Bibin A Chundatt via devaraj)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af63427c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
index a3c136c..4e7c29a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
@@ -62,9 +62,8 @@ public class TimelineAuthenticationFilterInitializer extends
FilterInitializer {
* Initializes {@link TimelineAuthenticationFilter}
* p
* Propagates to {@link TimelineAuthenticationFilter} configuration all YARN
- * configuration properties prefixed with
- * {@code yarn.timeline-service.authentication.}
- *
+ * configuration properties prefixed with {@value #PREFIX}
+ *
* @param container
* The filter container
* @param conf
hadoop git commit: MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 09639ac6b - 415dd6446
MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.
(cherry picked from commit bff67dfe2f811654ffb1bbcbd87509c185f452b6)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/415dd644
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/415dd644
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/415dd644
Branch: refs/heads/branch-2
Commit: 415dd64461f480abf5a93c30b21efd0cb38b35a9
Parents: 09639ac
Author: Devaraj K deva...@apache.org
Authored: Mon Jul 6 13:46:37 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Jul 6 13:47:54 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../apache/hadoop/mapred/ShuffleHandler.java| 3 +-
.../hadoop/mapred/TestShuffleHandler.java | 101 +++
3 files changed, 106 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/415dd644/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 7b21dd7..c2bd24c 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -275,6 +275,9 @@ Release 2.7.2 - UNRELEASED
BUG FIXES
+MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo
+if mapId is not in the cache. (zhihai xu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/415dd644/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index eedf42b..ee1be23 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -815,7 +815,8 @@ public class ShuffleHandler extends AuxiliaryService {
try {
MapOutputInfo info = mapOutputInfoMap.get(mapId);
if (info == null) {
-info = getMapOutputInfo(outputBasePathStr, mapId, reduceId, user);
+info = getMapOutputInfo(outputBasePathStr + mapId,
+mapId, reduceId, user);
}
lastMap =
sendMapOutput(ctx, ch, user, mapId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/415dd644/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 1a0688f..35d9bdd 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -601,6 +601,7 @@ public class TestShuffleHandler {
Assert.assertTrue((new String(byteArr)).contains(message));
} finally {
shuffleHandler.stop();
+ FileUtil.fullyDelete(absLogDir);
}
}
@@ -829,4 +830,104 @@ public class TestShuffleHandler {
conn.disconnect();
return rc;
}
+
+ @Test(timeout = 10)
+ public void testGetMapOutputInfo() throws Exception {
+final ArrayListThrowable failures = new ArrayListThrowable(1);
+Configuration conf = new Configuration();
+conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+File absLogDir = new File(target, TestShuffleHandler.class.
+getSimpleName() + LocDir).getAbsoluteFile();
+conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
+ApplicationId appId = ApplicationId.newInstance(12345, 1
hadoop git commit: MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 a6f6ba95e - c9c7a00a8
MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.
(cherry picked from commit bff67dfe2f811654ffb1bbcbd87509c185f452b6)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c9c7a00a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c9c7a00a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c9c7a00a
Branch: refs/heads/branch-2.7
Commit: c9c7a00a82016192e9c9a8b15d45f95e7790f820
Parents: a6f6ba9
Author: Devaraj K deva...@apache.org
Authored: Mon Jul 6 13:46:37 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Jul 6 13:48:44 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../apache/hadoop/mapred/ShuffleHandler.java| 3 +-
.../hadoop/mapred/TestShuffleHandler.java | 101 +++
3 files changed, 106 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9c7a00a/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 491d89a..e44624b 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -12,6 +12,9 @@ Release 2.7.2 - UNRELEASED
BUG FIXES
+MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo
+if mapId is not in the cache. (zhihai xu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9c7a00a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 6e069f1..d1a0778 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -806,7 +806,8 @@ public class ShuffleHandler extends AuxiliaryService {
try {
MapOutputInfo info = mapOutputInfoMap.get(mapId);
if (info == null) {
-info = getMapOutputInfo(outputBasePathStr, mapId, reduceId, user);
+info = getMapOutputInfo(outputBasePathStr + mapId,
+mapId, reduceId, user);
}
lastMap =
sendMapOutput(ctx, ch, user, mapId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c9c7a00a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 1a0688f..35d9bdd 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -601,6 +601,7 @@ public class TestShuffleHandler {
Assert.assertTrue((new String(byteArr)).contains(message));
} finally {
shuffleHandler.stop();
+ FileUtil.fullyDelete(absLogDir);
}
}
@@ -829,4 +830,104 @@ public class TestShuffleHandler {
conn.disconnect();
return rc;
}
+
+ @Test(timeout = 10)
+ public void testGetMapOutputInfo() throws Exception {
+final ArrayListThrowable failures = new ArrayListThrowable(1);
+Configuration conf = new Configuration();
+conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+File absLogDir = new File(target, TestShuffleHandler.class.
+getSimpleName() + LocDir).getAbsoluteFile();
+conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
+ApplicationId appId = ApplicationId.newInstance(12345, 1
hadoop git commit: MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/trunk 688617d6d - bff67dfe2
MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo if mapId is not in the cache. Contributed by zhihai xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bff67dfe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bff67dfe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bff67dfe
Branch: refs/heads/trunk
Commit: bff67dfe2f811654ffb1bbcbd87509c185f452b6
Parents: 688617d
Author: Devaraj K deva...@apache.org
Authored: Mon Jul 6 13:46:37 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Jul 6 13:46:37 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../apache/hadoop/mapred/ShuffleHandler.java| 3 +-
.../hadoop/mapred/TestShuffleHandler.java | 101 +++
3 files changed, 106 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bff67dfe/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 2f80615..2458403 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -542,6 +542,9 @@ Release 2.7.2 - UNRELEASED
BUG FIXES
+MAPREDUCE-6425. ShuffleHandler passes wrong base parameter to
getMapOutputInfo
+if mapId is not in the cache. (zhihai xu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bff67dfe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index eedf42b..ee1be23 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -815,7 +815,8 @@ public class ShuffleHandler extends AuxiliaryService {
try {
MapOutputInfo info = mapOutputInfoMap.get(mapId);
if (info == null) {
-info = getMapOutputInfo(outputBasePathStr, mapId, reduceId, user);
+info = getMapOutputInfo(outputBasePathStr + mapId,
+mapId, reduceId, user);
}
lastMap =
sendMapOutput(ctx, ch, user, mapId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bff67dfe/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
index 7053653..746071f 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/test/java/org/apache/hadoop/mapred/TestShuffleHandler.java
@@ -601,6 +601,7 @@ public class TestShuffleHandler {
Assert.assertTrue((new String(byteArr)).contains(message));
} finally {
shuffleHandler.stop();
+ FileUtil.fullyDelete(absLogDir);
}
}
@@ -829,4 +830,104 @@ public class TestShuffleHandler {
conn.disconnect();
return rc;
}
+
+ @Test(timeout = 10)
+ public void testGetMapOutputInfo() throws Exception {
+final ArrayListThrowable failures = new ArrayListThrowable(1);
+Configuration conf = new Configuration();
+conf.setInt(ShuffleHandler.SHUFFLE_PORT_CONFIG_KEY, 0);
+conf.setInt(ShuffleHandler.MAX_SHUFFLE_CONNECTIONS, 3);
+File absLogDir = new File(target, TestShuffleHandler.class.
+getSimpleName() + LocDir).getAbsoluteFile();
+conf.set(YarnConfiguration.NM_LOCAL_DIRS, absLogDir.getAbsolutePath());
+ApplicationId appId = ApplicationId.newInstance(12345, 1);
+String appAttemptId = attempt_12345_1_m_1_0;
+String user = randomUser
hadoop git commit: YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a null attempt. Contributed by nijel.
Repository: hadoop
Updated Branches:
refs/heads/trunk 7405c5979 - 80a68d605
YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a
null attempt. Contributed by nijel.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80a68d60
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80a68d60
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80a68d60
Branch: refs/heads/trunk
Commit: 80a68d60560e505b5f8e01969dc3c168a1e5a7f3
Parents: 7405c59
Author: Devaraj K deva...@apache.org
Authored: Wed Jul 1 19:03:44 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Wed Jul 1 19:03:44 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/AbstractYarnScheduler.java| 34 +
.../scheduler/TestAbstractYarnScheduler.java| 74 ++--
3 files changed, 91 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80a68d60/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8c07e48..4389e27 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -583,6 +583,9 @@ Release 2.8.0 - UNRELEASED
YARN-3823. Fix mismatch in default values for
yarn.scheduler.maximum-allocation-vcores property. (Ray Chiang via devaraj)
+YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a null
+attempt. (nijel via devaraj)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80a68d60/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index ae927f1..aad76fd 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -66,6 +66,7 @@ import
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import
org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.SettableFuture;
@@ -451,25 +452,30 @@ public abstract class AbstractYarnScheduler
new Timer().schedule(new TimerTask() {
@Override
public void run() {
-for (SchedulerApplicationT app : applications.values()) {
+clearPendingContainerCache();
+LOG.info(Release request cache is cleaned up);
+ }
+}, nmExpireInterval);
+ }
- T attempt = app.getCurrentAppAttempt();
- synchronized (attempt) {
-for (ContainerId containerId : attempt.getPendingRelease()) {
- RMAuditLogger.logFailure(
-app.getUser(),
+ @VisibleForTesting
+ public void clearPendingContainerCache() {
+for (SchedulerApplicationT app : applications.values()) {
+ T attempt = app.getCurrentAppAttempt();
+ if (attempt != null) {
+synchronized (attempt) {
+ for (ContainerId containerId : attempt.getPendingRelease()) {
+RMAuditLogger.logFailure(app.getUser(),
AuditConstants.RELEASE_CONTAINER,
-Unauthorized access or invalid container,
-Scheduler,
-Trying to release container not owned by app or with invalid
id.,
-attempt.getApplicationId(), containerId);
-}
-attempt.getPendingRelease().clear();
+Unauthorized access or invalid container, Scheduler,
+Trying to release container not owned by app
++ or with invalid id., attempt.getApplicationId(),
+containerId);
}
+ attempt.getPendingRelease().clear();
}
-LOG.info(Release request cache is cleaned up);
}
-}, nmExpireInterval);
+}
}
// clean up
hadoop git commit: YARN-3875. FSSchedulerNode#reserveResource() doesn't print Application Id properly in log. Contributed by Bibin A Chundatt.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 0d94a293a - e1ca88641
YARN-3875. FSSchedulerNode#reserveResource() doesn't print Application Id
properly in log. Contributed by Bibin A Chundatt.
(cherry picked from commit 37d7395773b5bb24aa522db38a2602df9a5ac184)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e1ca8864
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e1ca8864
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e1ca8864
Branch: refs/heads/branch-2
Commit: e1ca886419833d1499c1793c2392eb59313a2d82
Parents: 0d94a29
Author: Devaraj K deva...@apache.org
Authored: Thu Jul 2 10:20:31 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Jul 2 10:21:32 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../resourcemanager/scheduler/fair/FSSchedulerNode.java | 11 ++-
2 files changed, 9 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1ca8864/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index da725b9..ea9cc68 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -538,6 +538,9 @@ Release 2.8.0 - UNRELEASED
YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a null
attempt. (nijel via devaraj)
+YARN-3875. FSSchedulerNode#reserveResource() doesn't print Application Id
+properly in log. (Bibin A Chundatt via devaraj)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e1ca8864/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index be08dff..c86201a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -68,12 +68,13 @@ public class FSSchedulerNode extends SchedulerNode {
on node + this);
}
- LOG.info(Updated reserved container +
- container.getContainer().getId() + on node +
- this + for application + application);
+ LOG.info(Updated reserved container + container.getContainer().getId()
+ + on node + this + for application
+ + application.getApplicationId());
} else {
- LOG.info(Reserved container + container.getContainer().getId() +
- on node + this + for application + application);
+ LOG.info(Reserved container + container.getContainer().getId()
+ + on node + this + for application
+ + application.getApplicationId());
}
setReservedContainer(container);
this.reservedAppSchedulable = (FSAppAttempt) application;
hadoop git commit: YARN-3875. FSSchedulerNode#reserveResource() doesn't print Application Id properly in log. Contributed by Bibin A Chundatt.
Repository: hadoop
Updated Branches:
refs/heads/trunk f3796224b - 37d739577
YARN-3875. FSSchedulerNode#reserveResource() doesn't print Application Id
properly in log. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37d73957
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37d73957
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37d73957
Branch: refs/heads/trunk
Commit: 37d7395773b5bb24aa522db38a2602df9a5ac184
Parents: f379622
Author: Devaraj K deva...@apache.org
Authored: Thu Jul 2 10:20:31 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Jul 2 10:20:31 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../resourcemanager/scheduler/fair/FSSchedulerNode.java | 11 ++-
2 files changed, 9 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37d73957/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6839548..2009b47 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -586,6 +586,9 @@ Release 2.8.0 - UNRELEASED
YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a null
attempt. (nijel via devaraj)
+YARN-3875. FSSchedulerNode#reserveResource() doesn't print Application Id
+properly in log. (Bibin A Chundatt via devaraj)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37d73957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
index be08dff..c86201a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerNode.java
@@ -68,12 +68,13 @@ public class FSSchedulerNode extends SchedulerNode {
on node + this);
}
- LOG.info(Updated reserved container +
- container.getContainer().getId() + on node +
- this + for application + application);
+ LOG.info(Updated reserved container + container.getContainer().getId()
+ + on node + this + for application
+ + application.getApplicationId());
} else {
- LOG.info(Reserved container + container.getContainer().getId() +
- on node + this + for application + application);
+ LOG.info(Reserved container + container.getContainer().getId()
+ + on node + this + for application
+ + application.getApplicationId());
}
setReservedContainer(container);
this.reservedAppSchedulable = (FSAppAttempt) application;
hadoop git commit: YARN-3823. Fix mismatch in default values for yarn.scheduler.maximum-allocation-vcores property. Contributed by Ray Chiang.
Repository: hadoop Updated Branches: refs/heads/trunk 532e38cb7 - 7405c5979 YARN-3823. Fix mismatch in default values for yarn.scheduler.maximum-allocation-vcores property. Contributed by Ray Chiang. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7405c597 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7405c597 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7405c597 Branch: refs/heads/trunk Commit: 7405c59799ed1b8ad1a7c6f1b18fabf49d0b92b2 Parents: 532e38c Author: Devaraj K deva...@apache.org Authored: Wed Jul 1 11:11:26 2015 +0530 Committer: Devaraj K deva...@apache.org Committed: Wed Jul 1 11:11:26 2015 +0530 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop-yarn-common/src/main/resources/yarn-default.xml| 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/7405c597/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 5e9243d..8c07e48 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -580,6 +580,9 @@ Release 2.8.0 - UNRELEASED YARN-3768. ArrayIndexOutOfBoundsException with empty environment variables. (Zhihai Xu via gera) +YARN-3823. Fix mismatch in default values for +yarn.scheduler.maximum-allocation-vcores property. (Ray Chiang via devaraj) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/7405c597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 621198c..4987084 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -276,7 +276,7 @@ in terms of virtual CPU cores. Requests higher than this will throw a InvalidResourceRequestException./description nameyarn.scheduler.maximum-allocation-vcores/name -value32/value +value4/value /property property
hadoop git commit: YARN-3823. Fix mismatch in default values for yarn.scheduler.maximum-allocation-vcores property. Contributed by Ray Chiang.
Repository: hadoop Updated Branches: refs/heads/branch-2 fad478e7c - 3432017de YARN-3823. Fix mismatch in default values for yarn.scheduler.maximum-allocation-vcores property. Contributed by Ray Chiang. (cherry picked from commit 7405c59799ed1b8ad1a7c6f1b18fabf49d0b92b2) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3432017d Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3432017d Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3432017d Branch: refs/heads/branch-2 Commit: 3432017de00b205bdd1e6b59b587a25c8fba6817 Parents: fad478e Author: Devaraj K deva...@apache.org Authored: Wed Jul 1 11:11:26 2015 +0530 Committer: Devaraj K deva...@apache.org Committed: Wed Jul 1 11:12:43 2015 +0530 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../hadoop-yarn-common/src/main/resources/yarn-default.xml| 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3432017d/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f81b099..e4eddbb 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -532,6 +532,9 @@ Release 2.8.0 - UNRELEASED YARN-3768. ArrayIndexOutOfBoundsException with empty environment variables. (Zhihai Xu via gera) +YARN-3823. Fix mismatch in default values for +yarn.scheduler.maximum-allocation-vcores property. (Ray Chiang via devaraj) + Release 2.7.2 - UNRELEASED INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/3432017d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 621198c..4987084 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -276,7 +276,7 @@ in terms of virtual CPU cores. Requests higher than this will throw a InvalidResourceRequestException./description nameyarn.scheduler.maximum-allocation-vcores/name -value32/value +value4/value /property property
hadoop git commit: YARN-3859. LeafQueue doesn't print user properly for application add. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 1a8d162bc - e163c1e0d
YARN-3859. LeafQueue doesn't print user properly for application add.
Contributed by Varun Saxena.
(cherry picked from commit b543d1a390a67e5e92fea67d3a2635058c29e9da)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e163c1e0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e163c1e0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e163c1e0
Branch: refs/heads/branch-2
Commit: e163c1e0dabf2b8012cb6304351836f6dddb85a2
Parents: 1a8d162
Author: Devaraj K deva...@apache.org
Authored: Sun Jun 28 10:04:50 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Sun Jun 28 10:06:08 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e163c1e0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 971b9e7..5b440c5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -514,6 +514,9 @@ Release 2.8.0 - UNRELEASED
YARN-2871. TestRMRestart#testRMRestartGetApplicationList sometime fails in
trunk.
(zhihai xu via xgong)
+YARN-3859. LeafQueue doesn't print user properly for application add.
+(Varun Saxena via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e163c1e0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 8e39133..e5b44a6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -676,7 +676,8 @@ public class LeafQueue extends AbstractCSQueue {
LOG.info(Application added - +
appId: + application.getApplicationId() +
- user: + user + , + leaf-queue: + getQueueName() +
+ user: + application.getUser() + , +
+ leaf-queue: + getQueueName() +
#user-pending-applications: + user.getPendingApplications() +
#user-active-applications: + user.getActiveApplications() +
#queue-pending-applications: + getNumPendingApplications() +
hadoop git commit: YARN-3859. LeafQueue doesn't print user properly for application add. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/trunk 79ed0f959 - b543d1a39
YARN-3859. LeafQueue doesn't print user properly for application add.
Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b543d1a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b543d1a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b543d1a3
Branch: refs/heads/trunk
Commit: b543d1a390a67e5e92fea67d3a2635058c29e9da
Parents: 79ed0f9
Author: Devaraj K deva...@apache.org
Authored: Sun Jun 28 10:04:50 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Sun Jun 28 10:04:50 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b543d1a3/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e2bf706..cb6deb1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -562,6 +562,9 @@ Release 2.8.0 - UNRELEASED
YARN-2871. TestRMRestart#testRMRestartGetApplicationList sometime fails in
trunk.
(zhihai xu via xgong)
+YARN-3859. LeafQueue doesn't print user properly for application add.
+(Varun Saxena via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b543d1a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 8e39133..e5b44a6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -676,7 +676,8 @@ public class LeafQueue extends AbstractCSQueue {
LOG.info(Application added - +
appId: + application.getApplicationId() +
- user: + user + , + leaf-queue: + getQueueName() +
+ user: + application.getUser() + , +
+ leaf-queue: + getQueueName() +
#user-pending-applications: + user.getPendingApplications() +
#user-active-applications: + user.getActiveApplications() +
#queue-pending-applications: + getNumPendingApplications() +
hadoop git commit: YARN-3745. SerializedException should also try to instantiate internal exception with the default constructor. Contributed by Lavkesh Lahngir.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 3e20f595d - bb8a2cff2
YARN-3745. SerializedException should also try to instantiate internal
exception with the default constructor. Contributed by Lavkesh Lahngir.
(cherry picked from commit b381f88c71d18497deb35039372b1e9715d2c038)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bb8a2cff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bb8a2cff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bb8a2cff
Branch: refs/heads/branch-2
Commit: bb8a2cff2af4b09fedd6b27114ae99ace84e5aa8
Parents: 3e20f59
Author: Devaraj K deva...@apache.org
Authored: Thu Jun 25 17:38:11 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Jun 25 17:39:40 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../impl/pb/SerializedExceptionPBImpl.java | 26
.../impl/pb/TestSerializedExceptionPBImpl.java | 11 +
3 files changed, 35 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb8a2cff/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d4f23b6..2290b21 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -508,6 +508,9 @@ Release 2.8.0 - UNRELEASED
YARN-3826. Race condition in ResourceTrackerService leads to
wrong diagnostics messages. (Chengbing Liu via devaraj)
+YARN-3745. SerializedException should also try to instantiate internal
+exception with the default constructor. (Lavkesh Lahngir via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb8a2cff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
index e683fd5..fd9e170 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
@@ -158,15 +158,31 @@ public class SerializedExceptionPBImpl extends
SerializedException {
viaProto = false;
}
+ private static T extends Throwable T instantiateExceptionImpl(
+ String message, Class? extends T cls, Throwable cause)
+ throws NoSuchMethodException, InstantiationException,
+ IllegalAccessException, InvocationTargetException {
+Constructor? extends T cn;
+T ex = null;
+cn =
+cls.getConstructor(message == null ? new Class[0]
+: new Class[] {String.class});
+cn.setAccessible(true);
+ex = message == null ? cn.newInstance() : cn.newInstance(message);
+ex.initCause(cause);
+return ex;
+ }
+
private static T extends Throwable T instantiateException(
Class? extends T cls, String message, Throwable cause) {
-Constructor? extends T cn;
T ex = null;
try {
- cn = cls.getConstructor(String.class);
- cn.setAccessible(true);
- ex = cn.newInstance(message);
- ex.initCause(cause);
+ // Try constructor with String argument, if it fails, try default.
+ try {
+ex = instantiateExceptionImpl(message, cls, cause);
+ } catch (NoSuchMethodException e) {
+ex = instantiateExceptionImpl(null, cls, cause);
+ }
} catch (SecurityException e) {
throw new YarnRuntimeException(e);
} catch (NoSuchMethodException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bb8a2cff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
index b011e3e..ac7e40e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
+++
b/hadoop-yarn
hadoop git commit: YARN-3826. Race condition in ResourceTrackerService leads to wrong diagnostics messages. Contributed by Chengbing Liu.
Repository: hadoop
Updated Branches:
refs/heads/trunk a815cc157 - 57f1a01ed
YARN-3826. Race condition in ResourceTrackerService leads to wrong
diagnostics messages. Contributed by Chengbing Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/57f1a01e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/57f1a01e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/57f1a01e
Branch: refs/heads/trunk
Commit: 57f1a01eda80f44d3ffcbcb93c4ee290e274946a
Parents: a815cc1
Author: Devaraj K deva...@apache.org
Authored: Thu Jun 25 16:13:59 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Jun 25 16:13:59 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/utils/YarnServerBuilderUtils.java| 11 --
.../resourcemanager/ResourceTrackerService.java | 23 +---
3 files changed, 18 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57f1a01e/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9e70ec9..3d608fd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -553,6 +553,9 @@ Release 2.8.0 - UNRELEASED
YARN-3790. usedResource from rootQueue metrics may get stale data for FS
scheduler after recovering the container (Zhihai Xu via rohithsharmaks)
+YARN-3826. Race condition in ResourceTrackerService leads to
+wrong diagnostics messages. (Chengbing Liu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57f1a01e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
index 8bdff62..f333185 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
@@ -22,13 +22,11 @@ import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
-import org.apache.hadoop.yarn.util.Records;
/**
* Server Builder utilities to construct various objects.
@@ -39,6 +37,15 @@ public class YarnServerBuilderUtils {
private static final RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null);
+ public static NodeHeartbeatResponse newNodeHeartbeatResponse(
+ NodeAction action, String diagnosticsMessage) {
+NodeHeartbeatResponse response = recordFactory
+.newRecordInstance(NodeHeartbeatResponse.class);
+response.setNodeAction(action);
+response.setDiagnosticsMessage(diagnosticsMessage);
+return response;
+ }
+
public static NodeHeartbeatResponse newNodeHeartbeatResponse(int responseId,
NodeAction action, ListContainerId containersToCleanUp,
ListApplicationId applicationsToCleanUp,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57f1a01e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index aa37254..3c2c09b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main
hadoop git commit: YARN-3826. Race condition in ResourceTrackerService leads to wrong diagnostics messages. Contributed by Chengbing Liu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 4b8593caf - 3e20f595d
YARN-3826. Race condition in ResourceTrackerService leads to wrong
diagnostics messages. Contributed by Chengbing Liu.
(cherry picked from commit 57f1a01eda80f44d3ffcbcb93c4ee290e274946a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e20f595
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e20f595
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e20f595
Branch: refs/heads/branch-2
Commit: 3e20f595d4164c4dd9c958d7211c1f399a163992
Parents: 4b8593c
Author: Devaraj K deva...@apache.org
Authored: Thu Jun 25 16:13:59 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Jun 25 16:15:24 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/utils/YarnServerBuilderUtils.java| 11 --
.../resourcemanager/ResourceTrackerService.java | 23 +---
3 files changed, 18 insertions(+), 19 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e20f595/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2fb5075..d4f23b6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -505,6 +505,9 @@ Release 2.8.0 - UNRELEASED
YARN-3790. usedResource from rootQueue metrics may get stale data for FS
scheduler after recovering the container (Zhihai Xu via rohithsharmaks)
+YARN-3826. Race condition in ResourceTrackerService leads to
+wrong diagnostics messages. (Chengbing Liu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e20f595/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
index 8bdff62..f333185 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/YarnServerBuilderUtils.java
@@ -22,13 +22,11 @@ import java.util.List;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
-import org.apache.hadoop.yarn.util.Records;
/**
* Server Builder utilities to construct various objects.
@@ -39,6 +37,15 @@ public class YarnServerBuilderUtils {
private static final RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null);
+ public static NodeHeartbeatResponse newNodeHeartbeatResponse(
+ NodeAction action, String diagnosticsMessage) {
+NodeHeartbeatResponse response = recordFactory
+.newRecordInstance(NodeHeartbeatResponse.class);
+response.setNodeAction(action);
+response.setDiagnosticsMessage(diagnosticsMessage);
+return response;
+ }
+
public static NodeHeartbeatResponse newNodeHeartbeatResponse(int responseId,
NodeAction action, ListContainerId containersToCleanUp,
ListApplicationId applicationsToCleanUp,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e20f595/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index aa37254..3c2c09b 100644
---
a/hadoop-yarn-project
hadoop git commit: MAPREDUCE-6406. Update FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION_DEFAULT to match mapred-default.xml. Contributed by Ray Chiang.
Repository: hadoop
Updated Branches:
refs/heads/trunk 99271b762 - 41ae7768e
MAPREDUCE-6406. Update
FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION_DEFAULT to match
mapred-default.xml. Contributed by Ray Chiang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41ae7768
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41ae7768
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41ae7768
Branch: refs/heads/trunk
Commit: 41ae7768ebb8637c09c7f9733bcfe9dc43dbce69
Parents: 99271b7
Author: Devaraj K deva...@apache.org
Authored: Tue Jun 23 10:23:44 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Tue Jun 23 10:23:44 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java | 2 +-
.../src/main/resources/mapred-default.xml | 2 +-
3 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41ae7768/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index da1a2f3..0cefde0 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -200,6 +200,9 @@ Trunk (Unreleased)
MAPREDUCE-6396. TestPipeApplication fails by NullPointerException.
(Brahma Reddy Battula via aajisaka)
+MAPREDUCE-6406. Update
FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION_DEFAULT
+to match mapred-default.xml. (Ray Chiang via devaraj)
+
BREAKDOWN OF MAPREDUCE-2841 (NATIVE TASK) SUBTASKS
MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41ae7768/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
index 6e5d0a1..1e5d71a 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/output/FileOutputCommitter.java
@@ -63,7 +63,7 @@ public class FileOutputCommitter extends OutputCommitter {
mapreduce.fileoutputcommitter.marksuccessfuljobs;
public static final String FILEOUTPUTCOMMITTER_ALGORITHM_VERSION =
mapreduce.fileoutputcommitter.algorithm.version;
- public static final int FILEOUTPUTCOMMITTER_ALGORITHM_VERSION_DEFAULT = 1;
+ public static final int FILEOUTPUTCOMMITTER_ALGORITHM_VERSION_DEFAULT = 2;
private Path outputPath = null;
private Path workPath = null;
private final int algorithmVersion;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41ae7768/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 3789429..ba63c02 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1326,7 +1326,7 @@
value2/value
descriptionThe file output committer algorithm version
valid algorithm version number: 1 or 2
- default to 1, which is the original algorithm
+ default to 2, which is the original algorithm
In algorithm version 1,
hadoop git commit: MAPREDUCE-6405. NullPointerException in App Attempts page. Contributed by Siqi Li and Gera Shegalov.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c94579f84 - 5c3070dc1
MAPREDUCE-6405. NullPointerException in App Attempts page. Contributed by
Siqi Li and Gera Shegalov.
(cherry picked from commit 311a4179cc53a2754d1185b1cb10e6adb1b95d3d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c3070dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c3070dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c3070dc
Branch: refs/heads/branch-2
Commit: 5c3070dc169d4cfe37c1cb73d5c2573c84b3646c
Parents: c94579f
Author: Devaraj K deva...@apache.org
Authored: Sat Jun 20 10:35:04 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Sat Jun 20 10:37:42 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../mapreduce/v2/app/webapp/TaskPage.java | 82 +++-
2 files changed, 49 insertions(+), 36 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c3070dc/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index cc4bb9c..4e24e54 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -222,6 +222,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6373. The logger reports total input paths but it is referring
to input files. (Bibin A Chundatt via devaraj)
+MAPREDUCE-6405. NullPointerException in App Attempts page.
+(Siqi Li and Gera Shegalov via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c3070dc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
index 758b02c..d9f17c8 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
@@ -24,11 +24,14 @@ import static
org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+import java.util.EnumSet;
import java.util.Collection;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptInfo;
import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil;
@@ -48,7 +51,6 @@ public class TaskPage extends AppView {
static class AttemptsBlock extends HtmlBlock {
final App app;
final boolean enableUIActions;
-private String stateURLFormat;
@Inject
AttemptsBlock(App ctx, Configuration conf) {
@@ -66,37 +68,36 @@ public class TaskPage extends AppView {
return;
}
+ JobId jobId = app.getJob().getID();
if (enableUIActions) {
// Kill task attempt
-String appID = app.getJob().getID().getAppId().toString();
-String jobID = app.getJob().getID().toString();
-String taskID = app.getTask().getID().toString();
-stateURLFormat =
-String.format(/proxy/%s/ws/v1/mapreduce/jobs/%s/tasks/%s/
-+ attempts, appID, jobID, taskID) + /%s/state;
-
-String current =
-String.format(/proxy/%s/mapreduce/task/%s, appID, taskID);
StringBuilder script = new StringBuilder();
-script.append(function confirmAction(stateURL) {)
-.append( b = confirm(\Are you sure?\);)
-.append( if (b == true) {)
-.append( $.ajax({)
-.append( type: 'PUT',)
-.append( url: stateURL,)
-.append( contentType: 'application/json',)
-.append( data: '{\state\:\KILLED\}',)
-.append( dataType: 'json')
-.append( }).done(function(data){)
-.append( setTimeout(function(){)
-.append( location.href
hadoop git commit: MAPREDUCE-6405. NullPointerException in App Attempts page. Contributed by Siqi Li and Gera Shegalov.
Repository: hadoop
Updated Branches:
refs/heads/trunk d112d1832 - 311a4179c
MAPREDUCE-6405. NullPointerException in App Attempts page. Contributed by
Siqi Li and Gera Shegalov.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/311a4179
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/311a4179
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/311a4179
Branch: refs/heads/trunk
Commit: 311a4179cc53a2754d1185b1cb10e6adb1b95d3d
Parents: d112d18
Author: Devaraj K deva...@apache.org
Authored: Sat Jun 20 10:35:04 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Sat Jun 20 10:35:04 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../mapreduce/v2/app/webapp/TaskPage.java | 82 +++-
2 files changed, 49 insertions(+), 36 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/311a4179/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 7e57804..dfac028 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -483,6 +483,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6373. The logger reports total input paths but it is referring
to input files. (Bibin A Chundatt via devaraj)
+MAPREDUCE-6405. NullPointerException in App Attempts page.
+(Siqi Li and Gera Shegalov via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/311a4179/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
index 758b02c..d9f17c8 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/TaskPage.java
@@ -24,11 +24,14 @@ import static
org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.tableInit;
+import java.util.EnumSet;
import java.util.Collection;
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRConfig;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptState;
import org.apache.hadoop.mapreduce.v2.app.job.TaskAttempt;
import org.apache.hadoop.mapreduce.v2.app.webapp.dao.TaskAttemptInfo;
import org.apache.hadoop.mapreduce.v2.util.MRWebAppUtil;
@@ -48,7 +51,6 @@ public class TaskPage extends AppView {
static class AttemptsBlock extends HtmlBlock {
final App app;
final boolean enableUIActions;
-private String stateURLFormat;
@Inject
AttemptsBlock(App ctx, Configuration conf) {
@@ -66,37 +68,36 @@ public class TaskPage extends AppView {
return;
}
+ JobId jobId = app.getJob().getID();
if (enableUIActions) {
// Kill task attempt
-String appID = app.getJob().getID().getAppId().toString();
-String jobID = app.getJob().getID().toString();
-String taskID = app.getTask().getID().toString();
-stateURLFormat =
-String.format(/proxy/%s/ws/v1/mapreduce/jobs/%s/tasks/%s/
-+ attempts, appID, jobID, taskID) + /%s/state;
-
-String current =
-String.format(/proxy/%s/mapreduce/task/%s, appID, taskID);
StringBuilder script = new StringBuilder();
-script.append(function confirmAction(stateURL) {)
-.append( b = confirm(\Are you sure?\);)
-.append( if (b == true) {)
-.append( $.ajax({)
-.append( type: 'PUT',)
-.append( url: stateURL,)
-.append( contentType: 'application/json',)
-.append( data: '{\state\:\KILLED\}',)
-.append( dataType: 'json')
-.append( }).done(function(data){)
-.append( setTimeout(function(){)
-.append( location.href = ').append(current).append(';)
-.append( }, 1000
hadoop git commit: YARN-3824. Fix two minor nits in member variable properties of YarnConfiguration. Contributed by Ray Chiang.
Repository: hadoop
Updated Branches:
refs/heads/trunk 295d678be - 2ad668748
YARN-3824. Fix two minor nits in member variable properties of
YarnConfiguration. Contributed by Ray Chiang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ad66874
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ad66874
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ad66874
Branch: refs/heads/trunk
Commit: 2ad668748e2535e25e3b04656c29034d58074027
Parents: 295d678
Author: Devaraj K deva...@apache.org
Authored: Thu Jun 18 16:44:08 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Jun 18 16:44:08 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ad66874/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 243edb3..f00170e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -536,6 +536,9 @@ Release 2.8.0 - UNRELEASED
YARN-3617. Fix WindowsResourceCalculatorPlugin.getCpuFrequency()
returning always -1. (J.Andreina via devaraj)
+YARN-3824. Fix two minor nits in member variable properties
+of YarnConfiguration. (Ray Chiang via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ad66874/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 3ea1558..5d75a21 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -278,7 +278,7 @@ public class YarnConfiguration extends Configuration {
+ intermediate-data-encryption.enable;
@Private
- public static final Boolean DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION =
false;
+ public static final boolean DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION =
false;
/** The address of the RM admin interface.*/
public static final String RM_ADMIN_ADDRESS =
@@ -729,7 +729,7 @@ public class YarnConfiguration extends Configuration {
public static final String RM_PROXY_USER_PRIVILEGES_ENABLED = RM_PREFIX
+ proxy-user-privileges.enabled;
- public static boolean DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED = false;
+ public static final boolean DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED = false;
/**
* How many diagnostics/failure messages can be saved in RM for
hadoop git commit: YARN-3824. Fix two minor nits in member variable properties of YarnConfiguration. Contributed by Ray Chiang.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 80697e4f3 - 3b9698eca
YARN-3824. Fix two minor nits in member variable properties of
YarnConfiguration. Contributed by Ray Chiang.
(cherry picked from commit 2ad668748e2535e25e3b04656c29034d58074027)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3b9698ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3b9698ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3b9698ec
Branch: refs/heads/branch-2
Commit: 3b9698ecac75326d0cd88929189eaed782b9b8b2
Parents: 80697e4
Author: Devaraj K deva...@apache.org
Authored: Thu Jun 18 16:44:08 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Jun 18 16:46:33 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b9698ec/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b45e0c3..7a43bef 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -488,6 +488,9 @@ Release 2.8.0 - UNRELEASED
YARN-3617. Fix WindowsResourceCalculatorPlugin.getCpuFrequency()
returning always -1. (J.Andreina via devaraj)
+YARN-3824. Fix two minor nits in member variable properties
+of YarnConfiguration. (Ray Chiang via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b9698ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index e06ec1c..a777d7e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -278,7 +278,7 @@ public class YarnConfiguration extends Configuration {
+ intermediate-data-encryption.enable;
@Private
- public static final Boolean DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION =
false;
+ public static final boolean DEFAULT_YARN_INTERMEDIATE_DATA_ENCRYPTION =
false;
/** The address of the RM admin interface.*/
public static final String RM_ADMIN_ADDRESS =
@@ -729,7 +729,7 @@ public class YarnConfiguration extends Configuration {
public static final String RM_PROXY_USER_PRIVILEGES_ENABLED = RM_PREFIX
+ proxy-user-privileges.enabled;
- public static boolean DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED = false;
+ public static final boolean DEFAULT_RM_PROXY_USER_PRIVILEGES_ENABLED = false;
/**
* How many diagnostics/failure messages can be saved in RM for
hadoop git commit: MAPREDUCE-6373. The logger reports total input paths but it is referring to input files. Contributed by Bibin A Chundatt.
Repository: hadoop
Updated Branches:
refs/heads/trunk 74351af3b - 1babe50a2
MAPREDUCE-6373. The logger reports total input paths but it is referring
to input files. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1babe50a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1babe50a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1babe50a
Branch: refs/heads/trunk
Commit: 1babe50a2cbaae3c8165229347e743d0dc94e979
Parents: 74351af
Author: Devaraj K deva...@apache.org
Authored: Thu Jun 18 11:42:22 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Jun 18 11:42:22 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/mapred/FileInputFormat.java | 2 +-
.../org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java| 3 +--
3 files changed, 5 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1babe50a/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index a9f5053..1037091 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -477,6 +477,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6363. [NNBench] Lease mismatch error when running with multiple
mappers. (Brahma Reddy Battula via aajisaka)
+MAPREDUCE-6373. The logger reports total input paths but it is referring
+to input files. (Bibin A Chundatt via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1babe50a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
index c6cbd50..2c58ebe 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
@@ -253,7 +253,7 @@ public abstract class FileInputFormatK, V implements
InputFormatK, V {
LOG.debug(Time taken to get FileStatuses:
+ sw.now(TimeUnit.MILLISECONDS));
}
-LOG.info(Total input paths to process : + result.length);
+LOG.info(Total input files to process : + result.length);
return result;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1babe50a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
index f5cd5d1..0c5ede9 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.mapred.LocatedFileStatusFetcher;
-import org.apache.hadoop.mapred.SplitLocationInfo;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
@@ -287,7 +286,7 @@ public abstract class FileInputFormatK, V extends
InputFormatK, V {
LOG.debug(Time taken to get FileStatuses:
+ sw.now(TimeUnit.MILLISECONDS));
}
-LOG.info(Total input paths to process : + result.size());
+LOG.info(Total input files to process : + result.size());
return result;
}
hadoop git commit: MAPREDUCE-6373. The logger reports total input paths but it is referring to input files. Contributed by Bibin A Chundatt.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 8cac3888c - c4c7fd0bf
MAPREDUCE-6373. The logger reports total input paths but it is referring
to input files. Contributed by Bibin A Chundatt.
(cherry picked from commit 1babe50a2cbaae3c8165229347e743d0dc94e979)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4c7fd0b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4c7fd0b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4c7fd0b
Branch: refs/heads/branch-2
Commit: c4c7fd0bf1e53d046587cbf262e8f06a996abba5
Parents: 8cac388
Author: Devaraj K deva...@apache.org
Authored: Thu Jun 18 11:42:22 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Jun 18 11:45:11 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/mapred/FileInputFormat.java | 2 +-
.../org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java| 3 +--
3 files changed, 5 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4c7fd0b/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 6d6021a..c66535d 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -216,6 +216,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6363. [NNBench] Lease mismatch error when running with multiple
mappers. (Brahma Reddy Battula via aajisaka)
+MAPREDUCE-6373. The logger reports total input paths but it is referring
+to input files. (Bibin A Chundatt via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4c7fd0b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
index c6cbd50..2c58ebe 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/FileInputFormat.java
@@ -253,7 +253,7 @@ public abstract class FileInputFormatK, V implements
InputFormatK, V {
LOG.debug(Time taken to get FileStatuses:
+ sw.now(TimeUnit.MILLISECONDS));
}
-LOG.info(Total input paths to process : + result.length);
+LOG.info(Total input files to process : + result.length);
return result;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4c7fd0b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
index f5cd5d1..0c5ede9 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.mapred.LocatedFileStatusFetcher;
-import org.apache.hadoop.mapred.SplitLocationInfo;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.Job;
@@ -287,7 +286,7 @@ public abstract class FileInputFormatK, V extends
InputFormatK, V {
LOG.debug(Time taken to get FileStatuses:
+ sw.now(TimeUnit.MILLISECONDS));
}
-LOG.info(Total input paths to process : + result.size());
+LOG.info(Total input files to process : + result.size());
return result;
}
hadoop git commit: YARN-3617. Fix WindowsResourceCalculatorPlugin.getCpuFrequency() returning always -1. Contributed by J.Andreina.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 acc90ca17 - 2802c63ac
YARN-3617. Fix WindowsResourceCalculatorPlugin.getCpuFrequency() returning
always -1. Contributed by J.Andreina.
(cherry picked from commit 318d2cde7cb5c05a5f87c4ee967446bb60d28ae4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2802c63a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2802c63a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2802c63a
Branch: refs/heads/branch-2
Commit: 2802c63ac3b081e2e4e0caf0589d1f6e156461c2
Parents: acc90ca
Author: Devaraj K deva...@apache.org
Authored: Wed Jun 17 13:54:09 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Wed Jun 17 13:55:25 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2802c63a/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0ea3e3e..aa9e19a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -482,6 +482,9 @@ Release 2.8.0 - UNRELEASED
YARN-3714. AM proxy filter can not get RM webapp address from
yarn.resourcemanager.hostname.rm-id. (Masatake Iwasaki via xgong)
+YARN-3617. Fix WindowsResourceCalculatorPlugin.getCpuFrequency()
+returning always -1. (J.Andreina via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2802c63a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java
index 0e89118..cdbf525 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsResourceCalculatorPlugin.java
@@ -157,7 +157,7 @@ public class WindowsResourceCalculatorPlugin extends
ResourceCalculatorPlugin {
@Override
public long getCpuFrequency() {
refreshIfNeeded();
-return -1;
+return cpuFrequencyKhz;
}
/** {@inheritDoc} */
hadoop git commit: YARN-3148. Allow CORS related headers to passthrough in WebAppProxyServlet. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/trunk 318d2cde7 - ebb9a8251
YARN-3148. Allow CORS related headers to passthrough in
WebAppProxyServlet. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ebb9a825
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ebb9a825
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ebb9a825
Branch: refs/heads/trunk
Commit: ebb9a82519c622bb898e1eec5798c2298c726694
Parents: 318d2cd
Author: Devaraj K deva...@apache.org
Authored: Wed Jun 17 15:56:18 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Wed Jun 17 15:56:18 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../server/webproxy/WebAppProxyServlet.java | 5 +-
.../server/webproxy/TestWebAppProxyServlet.java | 57
3 files changed, 64 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebb9a825/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c793a31..afe76bd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -309,6 +309,9 @@ Release 2.8.0 - UNRELEASED
YARN-3789. Improve logs for LeafQueue#activateApplications().
(Bibin A Chundatt via devaraj)
+YARN-3148. Allow CORS related headers to passthrough in WebAppProxyServlet.
+(Varun Saxena via devaraj)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebb9a825/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
index b1e355d..33f36f0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
@@ -81,7 +81,10 @@ public class WebAppProxyServlet extends HttpServlet {
Accept-Encoding,
Accept-Language,
Accept-Charset,
-Content-Type));
+Content-Type,
+Origin,
+Access-Control-Request-Method,
+Access-Control-Request-Headers));
public static final String PROXY_USER_COOKIE_NAME = proxy-user;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ebb9a825/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
index 2a2ca2c..8e68c38 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/test/java/org/apache/hadoop/yarn/server/webproxy/TestWebAppProxyServlet.java
@@ -31,6 +31,7 @@ import java.net.HttpCookie;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
+import java.util.Enumeration;
import java.util.List;
import java.util.Map;
@@ -72,6 +73,10 @@ public class TestWebAppProxyServlet {
private static Server server;
private static int originalPort = 0;
+ private static int numberOfHeaders = 0;
+ private static final String UNKNOWN_HEADER = Unknown-Header;
+ private static boolean hasUnknownHeader = false;
+
/**
* Simple http server. Server should send answer with status 200
@@ -88,6 +93,9 @@ public class TestWebAppProxyServlet {
originalPort = server.getConnectors()[0].getLocalPort();
LOG.info(Running embedded servlet container at: http://localhost:;
+ originalPort);
+// This property needs to be set otherwise CORS Headers
hadoop git commit: YARN-3789. Improve logs for LeafQueue#activateApplications(). Contributed by Bibin A Chundatt.
Repository: hadoop
Updated Branches:
refs/heads/trunk 32e39d8a2 - b039e69bb
YARN-3789. Improve logs for LeafQueue#activateApplications(). Contributed
by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b039e69b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b039e69b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b039e69b
Branch: refs/heads/trunk
Commit: b039e69bb03accef485361af301fa59f03d08d6a
Parents: 32e39d8
Author: Devaraj K deva...@apache.org
Authored: Tue Jun 16 14:03:22 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Tue Jun 16 14:03:22 2015 +0530
--
hadoop-yarn-project/CHANGES.txt| 3 +++
.../scheduler/capacity/LeafQueue.java | 17 +
2 files changed, 12 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b039e69b/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e442244..d0d1e06 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -306,6 +306,9 @@ Release 2.8.0 - UNRELEASED
YARN-3785. Support for Resource as an argument during submitApp call in
MockRM
test class. (Sunil G via xgong)
+YARN-3789. Improve logs for LeafQueue#activateApplications().
+(Bibin A Chundatt via devaraj)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b039e69b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 15d3289..8e39133 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -56,7 +56,6 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.security.AccessType;
-import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import
org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import
org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
@@ -604,7 +603,7 @@ public class LeafQueue extends AbstractCSQueue {
for (IteratorFiCaSchedulerApp i=pendingApplications.iterator();
i.hasNext(); ) {
FiCaSchedulerApp application = i.next();
-
+ ApplicationId applicationId = application.getApplicationId();
// Check am resource limit
Resource amIfStarted =
Resources.add(application.getAMResource(), queueUsage.getAMUsed());
@@ -624,7 +623,9 @@ public class LeafQueue extends AbstractCSQueue {
single application in queue, it is likely set too low. +
skipping enforcement to allow at least one application to
start);
} else {
- LOG.info(not starting application as amIfStarted exceeds amLimit);
+ LOG.info(Not activating application + applicationId
+ + as amIfStarted: + amIfStarted + exceeds amLimit:
+ + amLimit);
continue;
}
}
@@ -645,8 +646,9 @@ public class LeafQueue extends AbstractCSQueue {
single application in queue for user, it is likely set too low.
+
skipping enforcement to allow at least one application to
start);
} else {
- LOG.info(not starting application as amIfStarted exceeds +
-userAmLimit);
+ LOG.info(Not activating application + applicationId
+ + for user: + user + as userAmIfStarted:
+ + userAmIfStarted + exceeds userAmLimit: + userAMLimit);
continue;
}
}
@@ -657,9 +659,8 @@ public class
hadoop git commit: YARN-3794. TestRMEmbeddedElector fails because of ambiguous LOG reference. Contributed by Chengbing Liu.
Repository: hadoop
Updated Branches:
refs/heads/trunk 83e8110f8 - d8dcfa98e
YARN-3794. TestRMEmbeddedElector fails because of ambiguous LOG reference.
Contributed by Chengbing Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d8dcfa98
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d8dcfa98
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d8dcfa98
Branch: refs/heads/trunk
Commit: d8dcfa98e3ca6a6fea414fd503589bb83b7a9c51
Parents: 83e8110
Author: Devaraj K deva...@apache.org
Authored: Fri Jun 12 13:42:49 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri Jun 12 13:42:49 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/TestRMEmbeddedElector.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8dcfa98/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index dc43ad8..f5780c9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -521,6 +521,9 @@ Release 2.8.0 - UNRELEASED
YARN-3778. Fix Yarn resourcemanger CLI usage. (Brahma Reddy Battula via
xgong)
+YARN-3794. TestRMEmbeddedElector fails because of ambiguous LOG reference.
+(Chengbing Liu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d8dcfa98/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
index 1b0bf7e..20b1c0e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
@@ -102,9 +102,9 @@ public class TestRMEmbeddedElector extends
ClientBaseWithFixes {
ServiceFailedException {
try {
callbackCalled.set(true);
-LOG.info(Callback called. Sleeping now);
+TestRMEmbeddedElector.LOG.info(Callback called. Sleeping
now);
Thread.sleep(delayMs);
-LOG.info(Sleep done);
+TestRMEmbeddedElector.LOG.info(Sleep done);
} catch (InterruptedException e) {
e.printStackTrace();
}
hadoop git commit: YARN-3794. TestRMEmbeddedElector fails because of ambiguous LOG reference. Contributed by Chengbing Liu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 f1ee2eaf8 - 3863342ca
YARN-3794. TestRMEmbeddedElector fails because of ambiguous LOG reference.
Contributed by Chengbing Liu.
(cherry picked from commit d8dcfa98e3ca6a6fea414fd503589bb83b7a9c51)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3863342c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3863342c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3863342c
Branch: refs/heads/branch-2
Commit: 3863342ca91494a311cf5524f0494a66a6fca842
Parents: f1ee2ea
Author: Devaraj K deva...@apache.org
Authored: Fri Jun 12 13:42:49 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri Jun 12 13:45:05 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/TestRMEmbeddedElector.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3863342c/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4a83ed5..30035c7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -473,6 +473,9 @@ Release 2.8.0 - UNRELEASED
YARN-3778. Fix Yarn resourcemanger CLI usage. (Brahma Reddy Battula via
xgong)
+YARN-3794. TestRMEmbeddedElector fails because of ambiguous LOG reference.
+(Chengbing Liu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3863342c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
index 1b0bf7e..20b1c0e 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMEmbeddedElector.java
@@ -102,9 +102,9 @@ public class TestRMEmbeddedElector extends
ClientBaseWithFixes {
ServiceFailedException {
try {
callbackCalled.set(true);
-LOG.info(Callback called. Sleeping now);
+TestRMEmbeddedElector.LOG.info(Callback called. Sleeping
now);
Thread.sleep(delayMs);
-LOG.info(Sleep done);
+TestRMEmbeddedElector.LOG.info(Sleep done);
} catch (InterruptedException e) {
e.printStackTrace();
}
hadoop git commit: MAPREDUCE-6350. JobHistory doesn't support fully-functional search. Contributed by Siqi Li.
Repository: hadoop
Updated Branches:
refs/heads/trunk 48ca23def - 6785661e5
MAPREDUCE-6350. JobHistory doesn't support fully-functional search.
Contributed by Siqi Li.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6785661e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6785661e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6785661e
Branch: refs/heads/trunk
Commit: 6785661e554114a4613b5fe7dabec9bfa80c41d9
Parents: 48ca23d
Author: Devaraj K deva...@apache.org
Authored: Wed Jun 10 12:40:29 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Wed Jun 10 12:40:29 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +++
.../jobhistory/JobHistoryEventHandler.java | 6 -
.../v2/jobhistory/FileNameIndexUtils.java | 17 -
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 7 ++
.../v2/jobhistory/TestFileNameIndexUtils.java | 26
.../src/main/resources/mapred-default.xml | 8 ++
6 files changed, 60 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 115fc1b..3c1884d 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -465,6 +465,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6388. Remove deprecation warnings from JobHistoryServer classes
(Ray Chiang via ozawa).
+MAPREDUCE-6350. JobHistory doesn't support fully-functional search.
+(Siqi Li via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index bf32888..35556a6 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
import org.apache.hadoop.security.UserGroupInformation;
@@ -1107,9 +1108,12 @@ public class JobHistoryEventHandler extends
AbstractService
if (mi.getHistoryFile() != null) {
Path historyFile = mi.getHistoryFile();
Path qualifiedLogFile = stagingDirFS.makeQualified(historyFile);
+int jobNameLimit =
+getConfig().getInt(JHAdminConfig.MR_HS_JOBNAME_LIMIT,
+JHAdminConfig.DEFAULT_MR_HS_JOBNAME_LIMIT);
String doneJobHistoryFileName =
getTempFileName(FileNameIndexUtils.getDoneFileName(mi
-.getJobIndexInfo()));
+.getJobIndexInfo(), jobNameLimit));
qualifiedDoneFile =
doneDirFS.makeQualified(new Path(doneDirPrefixPath,
doneJobHistoryFileName));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6785661e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
index 741da11..eb0c54c 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory
hadoop git commit: MAPREDUCE-6350. JobHistory doesn't support fully-functional search. Contributed by Siqi Li.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d21bc811d - 5426e3e92
MAPREDUCE-6350. JobHistory doesn't support fully-functional search.
Contributed by Siqi Li.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5426e3e9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5426e3e9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5426e3e9
Branch: refs/heads/branch-2
Commit: 5426e3e92999cb9e923bcd8a0c16e5c70268df5e
Parents: d21bc81
Author: Devaraj K deva...@apache.org
Authored: Wed Jun 10 12:43:31 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Wed Jun 10 12:43:31 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +++
.../jobhistory/JobHistoryEventHandler.java | 6 -
.../v2/jobhistory/FileNameIndexUtils.java | 17 -
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 7 ++
.../v2/jobhistory/TestFileNameIndexUtils.java | 26
.../src/main/resources/mapred-default.xml | 8 ++
6 files changed, 60 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5426e3e9/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 3f9ec14..d2451fc 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -207,6 +207,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6388. Remove deprecation warnings from JobHistoryServer classes
(Ray Chiang via ozawa).
+MAPREDUCE-6350. JobHistory doesn't support fully-functional search.
+(Siqi Li via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5426e3e9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index bf32888..35556a6 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
+import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo;
import org.apache.hadoop.security.UserGroupInformation;
@@ -1107,9 +1108,12 @@ public class JobHistoryEventHandler extends
AbstractService
if (mi.getHistoryFile() != null) {
Path historyFile = mi.getHistoryFile();
Path qualifiedLogFile = stagingDirFS.makeQualified(historyFile);
+int jobNameLimit =
+getConfig().getInt(JHAdminConfig.MR_HS_JOBNAME_LIMIT,
+JHAdminConfig.DEFAULT_MR_HS_JOBNAME_LIMIT);
String doneJobHistoryFileName =
getTempFileName(FileNameIndexUtils.getDoneFileName(mi
-.getJobIndexInfo()));
+.getJobIndexInfo(), jobNameLimit));
qualifiedDoneFile =
doneDirFS.makeQualified(new Path(doneDirPrefixPath,
doneJobHistoryFileName));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5426e3e9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/FileNameIndexUtils.java
index 741da11..eb0c54c 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory
hadoop git commit: MAPREDUCE-6389. Fix BaileyBorweinPlouffe CLI usage message. Contributed by Brahma Reddy Battula.
Repository: hadoop
Updated Branches:
refs/heads/trunk 6785661e5 - c7729efee
MAPREDUCE-6389. Fix BaileyBorweinPlouffe CLI usage message. Contributed by
Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7729efe
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7729efe
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7729efe
Branch: refs/heads/trunk
Commit: c7729efee8727b59f2c78cd5a3ad23fa84139068
Parents: 6785661
Author: Devaraj K deva...@apache.org
Authored: Wed Jun 10 21:05:10 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Wed Jun 10 21:05:10 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7729efe/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 3c1884d..f475454 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -468,6 +468,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6350. JobHistory doesn't support fully-functional search.
(Siqi Li via devaraj)
+MAPREDUCE-6389. Fix BaileyBorweinPlouffe CLI usage message. (Brahma Reddy
Battula
+via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7729efe/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
index 7c023cf..7e98d7d 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
@@ -400,7 +400,7 @@ public class BaileyBorweinPlouffe extends Configured
implements Tool {
*/
public int run(String[] args) throws IOException {
if (args.length != 4) {
- System.err.println(Usage: java + getClass().getName()
+ System.err.println(Usage: bbp
+ startDigit nDigits nMaps workingDir);
ToolRunner.printGenericCommandUsage(System.err);
return -1;
hadoop git commit: MAPREDUCE-6389. Fix BaileyBorweinPlouffe CLI usage message. Contributed by Brahma Reddy Battula.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 5426e3e92 - b117527bf
MAPREDUCE-6389. Fix BaileyBorweinPlouffe CLI usage message. Contributed by
Brahma Reddy Battula.
(cherry picked from commit c7729efee8727b59f2c78cd5a3ad23fa84139068)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b117527b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b117527b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b117527b
Branch: refs/heads/branch-2
Commit: b117527bf35938494e5063327bdd4bf9404366e2
Parents: 5426e3e
Author: Devaraj K deva...@apache.org
Authored: Wed Jun 10 21:05:10 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Wed Jun 10 21:06:35 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b117527b/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index d2451fc..6776490 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -210,6 +210,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6350. JobHistory doesn't support fully-functional search.
(Siqi Li via devaraj)
+MAPREDUCE-6389. Fix BaileyBorweinPlouffe CLI usage message. (Brahma Reddy
Battula
+via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b117527b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
index 7c023cf..7e98d7d 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/BaileyBorweinPlouffe.java
@@ -400,7 +400,7 @@ public class BaileyBorweinPlouffe extends Configured
implements Tool {
*/
public int run(String[] args) throws IOException {
if (args.length != 4) {
- System.err.println(Usage: java + getClass().getName()
+ System.err.println(Usage: bbp
+ startDigit nDigits nMaps workingDir);
ToolRunner.printGenericCommandUsage(System.err);
return -1;
hadoop git commit: YARN-3780. Should use equals when compare Resource in RMNodeImpl#ReconnectNodeTransition. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 953c4a049 - 2a01c01af
YARN-3780. Should use equals when compare Resource in
RMNodeImpl#ReconnectNodeTransition. Contributed by zhihai xu.
(cherry picked from commit c7ee6c151c5771043a6de3b8a951cea13f59dd7b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a01c01a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a01c01a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a01c01a
Branch: refs/heads/branch-2
Commit: 2a01c01af7d5d5bc2bf7e0223bffde6c081b4c21
Parents: 953c4a0
Author: Devaraj K deva...@apache.org
Authored: Mon Jun 8 11:54:55 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Jun 8 11:56:58 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a01c01a/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0fb8b04..e0272b5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -453,6 +453,9 @@ Release 2.8.0 - UNRELEASED
YARN-3655. FairScheduler: potential livelock due to maxAMShare limitation
and container reservation. (Zhihai Xu via kasha)
+YARN-3780. Should use equals when compare Resource in
RMNodeImpl#ReconnectNodeTransition.
+(zhihai xu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a01c01a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 1263692..8a810cb 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -622,7 +622,8 @@ public class RMNodeImpl implements RMNode,
EventHandlerRMNodeEvent {
rmNode.httpPort = newNode.getHttpPort();
rmNode.httpAddress = newNode.getHttpAddress();
boolean isCapabilityChanged = false;
-if (rmNode.getTotalCapability() != newNode.getTotalCapability()) {
+if (!rmNode.getTotalCapability().equals(
+newNode.getTotalCapability())) {
rmNode.totalCapability = newNode.getTotalCapability();
isCapabilityChanged = true;
}
hadoop git commit: YARN-3780. Should use equals when compare Resource in RMNodeImpl#ReconnectNodeTransition. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/trunk a6cb4894b - c7ee6c151
YARN-3780. Should use equals when compare Resource in
RMNodeImpl#ReconnectNodeTransition. Contributed by zhihai xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7ee6c15
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7ee6c15
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7ee6c15
Branch: refs/heads/trunk
Commit: c7ee6c151c5771043a6de3b8a951cea13f59dd7b
Parents: a6cb489
Author: Devaraj K deva...@apache.org
Authored: Mon Jun 8 11:54:55 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Jun 8 11:54:55 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ee6c15/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d90433d..59e3509 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -501,6 +501,9 @@ Release 2.8.0 - UNRELEASED
YARN-3655. FairScheduler: potential livelock due to maxAMShare limitation
and container reservation. (Zhihai Xu via kasha)
+YARN-3780. Should use equals when compare Resource in
RMNodeImpl#ReconnectNodeTransition.
+(zhihai xu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7ee6c15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 1263692..8a810cb 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -622,7 +622,8 @@ public class RMNodeImpl implements RMNode,
EventHandlerRMNodeEvent {
rmNode.httpPort = newNode.getHttpPort();
rmNode.httpAddress = newNode.getHttpAddress();
boolean isCapabilityChanged = false;
-if (rmNode.getTotalCapability() != newNode.getTotalCapability()) {
+if (!rmNode.getTotalCapability().equals(
+newNode.getTotalCapability())) {
rmNode.totalCapability = newNode.getTotalCapability();
isCapabilityChanged = true;
}
hadoop git commit: YARN-3747. TestLocalDirsHandlerService should delete the created test directory logDir2. Contributed by David Moore.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 2a01c01af - 8c643e3bf
YARN-3747. TestLocalDirsHandlerService should delete the created test
directory logDir2. Contributed by David Moore.
(cherry picked from commit 126321eded7dc38c1eef2cfde9365404c924a5cb)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8c643e3b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8c643e3b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8c643e3b
Branch: refs/heads/branch-2
Commit: 8c643e3bf9e0a70229aee79fb5244a19b32fef00
Parents: 2a01c01
Author: Devaraj K deva...@apache.org
Authored: Mon Jun 8 15:32:13 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Jun 8 15:33:38 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/nodemanager/TestLocalDirsHandlerService.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c643e3b/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e0272b5..05cab3d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -456,6 +456,9 @@ Release 2.8.0 - UNRELEASED
YARN-3780. Should use equals when compare Resource in
RMNodeImpl#ReconnectNodeTransition.
(zhihai xu via devaraj)
+YARN-3747. TestLocalDirsHandlerService should delete the created test
directory logDir2.
+(David Moore via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8c643e3b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
index a045e62..c61d1f0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
@@ -144,7 +144,7 @@ public class TestLocalDirsHandlerService {
FileUtils.deleteDirectory(new File(localDir1));
FileUtils.deleteDirectory(new File(localDir2));
FileUtils.deleteDirectory(new File(logDir1));
-FileUtils.deleteDirectory(new File(logDir1));
+FileUtils.deleteDirectory(new File(logDir2));
dirSvc.close();
}
}
hadoop git commit: YARN-3747. TestLocalDirsHandlerService should delete the created test directory logDir2. Contributed by David Moore.
Repository: hadoop
Updated Branches:
refs/heads/trunk c7ee6c151 - 126321ede
YARN-3747. TestLocalDirsHandlerService should delete the created test
directory logDir2. Contributed by David Moore.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/126321ed
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/126321ed
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/126321ed
Branch: refs/heads/trunk
Commit: 126321eded7dc38c1eef2cfde9365404c924a5cb
Parents: c7ee6c1
Author: Devaraj K deva...@apache.org
Authored: Mon Jun 8 15:32:13 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Jun 8 15:32:13 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/nodemanager/TestLocalDirsHandlerService.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/126321ed/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 59e3509..f393cad 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -504,6 +504,9 @@ Release 2.8.0 - UNRELEASED
YARN-3780. Should use equals when compare Resource in
RMNodeImpl#ReconnectNodeTransition.
(zhihai xu via devaraj)
+YARN-3747. TestLocalDirsHandlerService should delete the created test
directory logDir2.
+(David Moore via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/126321ed/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
index a045e62..c61d1f0 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLocalDirsHandlerService.java
@@ -144,7 +144,7 @@ public class TestLocalDirsHandlerService {
FileUtils.deleteDirectory(new File(localDir1));
FileUtils.deleteDirectory(new File(localDir2));
FileUtils.deleteDirectory(new File(logDir1));
-FileUtils.deleteDirectory(new File(logDir1));
+FileUtils.deleteDirectory(new File(logDir2));
dirSvc.close();
}
}
hadoop git commit: MAPREDUCE-6383. Pi job (QuasiMonteCarlo) should not try to read the results file if its job fails. Contributed by Harsh J.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 7f075f2b7 - 37ad3b3fe
MAPREDUCE-6383. Pi job (QuasiMonteCarlo) should not try to read the
results file if its job fails. Contributed by Harsh J.
(cherry picked from commit 09b8241e6e369ee00241efcc2e048e710dbcccd1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/37ad3b3f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/37ad3b3f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/37ad3b3f
Branch: refs/heads/branch-2
Commit: 37ad3b3fef233a118ecf898e1f0ae54ef9f158c9
Parents: 7f075f2
Author: Devaraj K deva...@apache.org
Authored: Fri Jun 5 21:22:47 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri Jun 5 21:23:48 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java| 4
2 files changed, 7 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37ad3b3f/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 346d5f1..30a2666 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -76,6 +76,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6174. Combine common stream code into parent class for
InMemoryMapOutput and OnDiskMapOutput. (Eric Payne via gera)
+MAPREDUCE-6383. Pi job (QuasiMonteCarlo) should not try to read the
+results file if its job fails. (Harsh J via devaraj)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/37ad3b3f/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
index 25dee6b..1a0c372 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
@@ -304,6 +304,10 @@ public class QuasiMonteCarlo extends Configured implements
Tool {
System.out.println(Starting Job);
final long startTime = System.currentTimeMillis();
job.waitForCompletion(true);
+ if (!job.isSuccessful()) {
+System.out.println(Job + job.getJobID() + failed!);
+System.exit(1);
+ }
final double duration = (System.currentTimeMillis() - startTime)/1000.0;
System.out.println(Job Finished in + duration + seconds);
hadoop git commit: MAPREDUCE-6383. Pi job (QuasiMonteCarlo) should not try to read the results file if its job fails. Contributed by Harsh J.
Repository: hadoop
Updated Branches:
refs/heads/trunk 790a86176 - 09b8241e6
MAPREDUCE-6383. Pi job (QuasiMonteCarlo) should not try to read the
results file if its job fails. Contributed by Harsh J.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09b8241e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09b8241e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09b8241e
Branch: refs/heads/trunk
Commit: 09b8241e6e369ee00241efcc2e048e710dbcccd1
Parents: 790a861
Author: Devaraj K deva...@apache.org
Authored: Fri Jun 5 21:22:47 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri Jun 5 21:22:47 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java| 4
2 files changed, 7 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09b8241e/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 2ffa541..1813c31 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -337,6 +337,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6174. Combine common stream code into parent class for
InMemoryMapOutput and OnDiskMapOutput. (Eric Payne via gera)
+MAPREDUCE-6383. Pi job (QuasiMonteCarlo) should not try to read the
+results file if its job fails. (Harsh J via devaraj)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09b8241e/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
index 25dee6b..1a0c372 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/QuasiMonteCarlo.java
@@ -304,6 +304,10 @@ public class QuasiMonteCarlo extends Configured implements
Tool {
System.out.println(Starting Job);
final long startTime = System.currentTimeMillis();
job.waitForCompletion(true);
+ if (!job.isSuccessful()) {
+System.out.println(Job + job.getJobID() + failed!);
+System.exit(1);
+ }
final double duration = (System.currentTimeMillis() - startTime)/1000.0;
System.out.println(Job Finished in + duration + seconds);
hadoop git commit: MAPREDUCE-6377. JHS sorting on state column not working in webUi. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/trunk b2540f486 - 790a86176
MAPREDUCE-6377. JHS sorting on state column not working in webUi.
Contributed by zhihai xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/790a8617
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/790a8617
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/790a8617
Branch: refs/heads/trunk
Commit: 790a861766dcd60212d83f444f2f96d3acf20a94
Parents: b2540f4
Author: Devaraj K deva...@apache.org
Authored: Fri Jun 5 15:50:16 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri Jun 5 15:50:16 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +++
.../java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java | 5 +++--
2 files changed, 6 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/790a8617/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index fd40ad2..2ffa541 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -450,6 +450,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6382. Don't escape HTML links in Diagnostics in JHS job overview.
(Siqi Li via gera)
+MAPREDUCE-6377. JHS sorting on state column not working in webUi.
+(zhihai xu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/790a8617/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java
index 59b7aa6..229bbb0 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java
@@ -87,10 +87,11 @@ public class HsView extends TwoColumnLayout {
append(, bProcessing: true).
// Sort by id upon page load
-append(, aaSorting: [[2, 'desc']]).
+append(, aaSorting: [[3, 'desc']]).
append(, aoColumnDefs:[).
// Maps Total, Maps Completed, Reduces Total and Reduces Completed
-append({'sType':'numeric', 'bSearchable': false, 'aTargets': [ 7, 8,
9, 10 ] }).
+append({'sType':'numeric', 'bSearchable': false +
+, 'aTargets': [ 8, 9, 10, 11 ] }).
append(]}).
toString();
}
hadoop git commit: MAPREDUCE-6377. JHS sorting on state column not working in webUi. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 320e4cc0e - 7f075f2b7
MAPREDUCE-6377. JHS sorting on state column not working in webUi.
Contributed by zhihai xu.
(cherry picked from commit 790a861766dcd60212d83f444f2f96d3acf20a94)
Conflicts:
hadoop-mapreduce-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7f075f2b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7f075f2b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7f075f2b
Branch: refs/heads/branch-2
Commit: 7f075f2b7504f80f3896b82a23a786928010c819
Parents: 320e4cc
Author: Devaraj K deva...@apache.org
Authored: Fri Jun 5 15:50:16 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri Jun 5 16:03:02 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +++
.../java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java | 5 +++--
2 files changed, 6 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f075f2b/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index d215fc2..346d5f1 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -192,6 +192,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-5232. Add a configuration to be able to log classpath and other
system properties on mapreduce JVMs startup. (Sangjin Lee via vinodkv)
+MAPREDUCE-6377. JHS sorting on state column not working in webUi.
+(zhihai xu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f075f2b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java
index 59b7aa6..229bbb0 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/webapp/HsView.java
@@ -87,10 +87,11 @@ public class HsView extends TwoColumnLayout {
append(, bProcessing: true).
// Sort by id upon page load
-append(, aaSorting: [[2, 'desc']]).
+append(, aaSorting: [[3, 'desc']]).
append(, aoColumnDefs:[).
// Maps Total, Maps Completed, Reduces Total and Reduces Completed
-append({'sType':'numeric', 'bSearchable': false, 'aTargets': [ 7, 8,
9, 10 ] }).
+append({'sType':'numeric', 'bSearchable': false +
+, 'aTargets': [ 8, 9, 10, 11 ] }).
append(]}).
toString();
}
hadoop git commit: YARN-3722. Merge multiple TestWebAppUtils into o.a.h.yarn.webapp.util.TestWebAppUtils. Contributed by Masatake Iwasaki.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 25db34127 - 7116141d7
YARN-3722. Merge multiple TestWebAppUtils into
o.a.h.yarn.webapp.util.TestWebAppUtils. Contributed by Masatake Iwasaki.
(cherry picked from commit 7e509f58439b1089462f51f9a0f9782faec7d198)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7116141d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7116141d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7116141d
Branch: refs/heads/branch-2
Commit: 7116141d7da96551bb04a7949a7505460ea96c4a
Parents: 25db341
Author: Devaraj K deva...@apache.org
Authored: Thu May 28 15:35:06 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu May 28 15:36:13 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/util/TestWebAppUtils.java | 81
.../yarn/webapp/util/TestWebAppUtils.java | 52 +
3 files changed, 55 insertions(+), 81 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7116141d/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3eb1f3e..eceb153 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -229,6 +229,9 @@ Release 2.8.0 - UNRELEASED
YARN-3700. Made generic history service load a number of latest
applications
according to the parameter or the configuration. (Xuan Gong via zjshen)
+YARN-3722. Merge multiple TestWebAppUtils into
o.a.h.yarn.webapp.util.TestWebAppUtils.
+(Masatake Iwasaki via devaraj)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7116141d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
deleted file mode 100644
index 169787f..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * License); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an AS IS BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.util;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.Map;
-
-public class TestWebAppUtils {
-
- private static final String RM1_NODE_ID = rm1;
- private static final String RM2_NODE_ID = rm2;
-
- // Because WebAppUtils#getResolvedAddress tries to resolve the hostname, we
add a static mapping for dummy hostnames
- // to make this test run anywhere without having to give some resolvable
hostnames
- private static String dummyHostNames[] = {host1, host2, host3};
- private static final String anyIpAddress = 1.2.3.4;
- private static MapString, String savedStaticResolution = new HashMap();
-
- @BeforeClass
- public static void initializeDummyHostnameResolution() throws Exception {
-String previousIpAddress;
-for (String hostName : dummyHostNames) {
- if (null != (previousIpAddress =
NetUtils.getStaticResolution(hostName))) {
-savedStaticResolution.put(hostName, previousIpAddress);
- }
- NetUtils.addStaticResolution(hostName, anyIpAddress);
-}
- }
-
- @AfterClass
- public static void restoreDummyHostnameResolution() throws Exception {
-for (Map.EntryString, String
hadoop git commit: YARN-3722. Merge multiple TestWebAppUtils into o.a.h.yarn.webapp.util.TestWebAppUtils. Contributed by Masatake Iwasaki.
Repository: hadoop
Updated Branches:
refs/heads/trunk 50eeea130 - 7e509f584
YARN-3722. Merge multiple TestWebAppUtils into
o.a.h.yarn.webapp.util.TestWebAppUtils. Contributed by Masatake Iwasaki.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e509f58
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e509f58
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e509f58
Branch: refs/heads/trunk
Commit: 7e509f58439b1089462f51f9a0f9782faec7d198
Parents: 50eeea1
Author: Devaraj K deva...@apache.org
Authored: Thu May 28 15:35:06 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu May 28 15:35:06 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/util/TestWebAppUtils.java | 81
.../yarn/webapp/util/TestWebAppUtils.java | 52 +
3 files changed, 55 insertions(+), 81 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e509f58/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1d22076..8b8f390 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -277,6 +277,9 @@ Release 2.8.0 - UNRELEASED
YARN-3700. Made generic history service load a number of latest
applications
according to the parameter or the configuration. (Xuan Gong via zjshen)
+YARN-3722. Merge multiple TestWebAppUtils into
o.a.h.yarn.webapp.util.TestWebAppUtils.
+(Masatake Iwasaki via devaraj)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e509f58/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
deleted file mode 100644
index 169787f..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestWebAppUtils.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * License); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an AS IS BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.util;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.net.UnknownHostException;
-import java.util.HashMap;
-import java.util.Map;
-
-public class TestWebAppUtils {
-
- private static final String RM1_NODE_ID = rm1;
- private static final String RM2_NODE_ID = rm2;
-
- // Because WebAppUtils#getResolvedAddress tries to resolve the hostname, we
add a static mapping for dummy hostnames
- // to make this test run anywhere without having to give some resolvable
hostnames
- private static String dummyHostNames[] = {host1, host2, host3};
- private static final String anyIpAddress = 1.2.3.4;
- private static MapString, String savedStaticResolution = new HashMap();
-
- @BeforeClass
- public static void initializeDummyHostnameResolution() throws Exception {
-String previousIpAddress;
-for (String hostName : dummyHostNames) {
- if (null != (previousIpAddress =
NetUtils.getStaticResolution(hostName))) {
-savedStaticResolution.put(hostName, previousIpAddress);
- }
- NetUtils.addStaticResolution(hostName, anyIpAddress);
-}
- }
-
- @AfterClass
- public static void restoreDummyHostnameResolution() throws Exception {
-for (Map.EntryString, String hostnameToIpEntry :
savedStaticResolution.entrySet
hadoop git commit: YARN-3646. Applications are getting stuck some times in case of retry policy forever. Contributed by Raju Bairishetti.
Repository: hadoop
Updated Branches:
refs/heads/trunk a5def5808 - 0305316d6
YARN-3646. Applications are getting stuck some times in case of retry
policy forever. Contributed by Raju Bairishetti.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0305316d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0305316d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0305316d
Branch: refs/heads/trunk
Commit: 0305316d6932e6f1a05021354d77b6934e57e171
Parents: a5def58
Author: Devaraj K deva...@apache.org
Authored: Thu May 21 20:14:44 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu May 21 20:14:44 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../yarn/client/api/impl/TestYarnClient.java| 32
.../org/apache/hadoop/yarn/client/RMProxy.java | 15 -
3 files changed, 43 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0305316d/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d1d2258..e5a9ee9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -538,6 +538,9 @@ Release 2.7.1 - UNRELEASED
YARN-3694. Fix dead link for TimelineServer REST API.
(Jagadesh Kiran N via aajisaka)
+YARN-3646. Applications are getting stuck some times in case of retry
+policy forever. (Raju Bairishetti via devaraj)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0305316d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 511fa4a..bc40b9a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -1265,4 +1265,36 @@ public class TestYarnClient {
ReservationSystemTestUtil.reservationQ);
return request;
}
+
+ @Test(timeout = 3, expected = ApplicationNotFoundException.class)
+ public void testShouldNotRetryForeverForNonNetworkExceptions() throws
Exception {
+YarnConfiguration conf = new YarnConfiguration();
+conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, -1);
+
+ResourceManager rm = null;
+YarnClient yarnClient = null;
+try {
+ // start rm
+ rm = new ResourceManager();
+ rm.init(conf);
+ rm.start();
+
+ yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(conf);
+ yarnClient.start();
+
+ // create invalid application id
+ ApplicationId appId = ApplicationId.newInstance(1430126768L, 10645);
+
+ // RM should throw ApplicationNotFoundException exception
+ yarnClient.getApplicationReport(appId);
+} finally {
+ if (yarnClient != null) {
+yarnClient.stop();
+ }
+ if (rm != null) {
+rm.stop();
+ }
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0305316d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
index fa8d642..28628f3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
@@ -224,19 +224,20 @@ public class RMProxyT {
failoverSleepBaseMs, failoverSleepMaxMs);
}
-if (waitForEver) {
- return RetryPolicies.RETRY_FOREVER;
-}
-
if (rmConnectionRetryIntervalMS 0) {
throw new YarnRuntimeException(Invalid Configuration. +
YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
should not be negative.);
}
-RetryPolicy retryPolicy =
-RetryPolicies.retryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS,
-rmConnectionRetryIntervalMS
hadoop git commit: YARN-3646. Applications are getting stuck some times in case of retry policy forever. Contributed by Raju Bairishetti.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 7a8e076ff - b68c338b1
YARN-3646. Applications are getting stuck some times in case of retry
policy forever. Contributed by Raju Bairishetti.
(cherry picked from commit 0305316d6932e6f1a05021354d77b6934e57e171)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b68c338b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b68c338b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b68c338b
Branch: refs/heads/branch-2.7
Commit: b68c338b17ea8cf98af9b72f8f05b923ccdd2f26
Parents: 7a8e076
Author: Devaraj K deva...@apache.org
Authored: Thu May 21 20:14:44 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu May 21 20:16:53 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../yarn/client/api/impl/TestYarnClient.java| 32
.../org/apache/hadoop/yarn/client/RMProxy.java | 15 -
3 files changed, 43 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b68c338b/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4b325ad..6b95c41 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -115,6 +115,9 @@ Release 2.7.1 - UNRELEASED
YARN-3694. Fix dead link for TimelineServer REST API.
(Jagadesh Kiran N via aajisaka)
+YARN-3646. Applications are getting stuck some times in case of retry
+policy forever. (Raju Bairishetti via devaraj)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b68c338b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index de669f2..738b0a8 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -1214,4 +1214,36 @@ public class TestYarnClient {
ReservationSystemTestUtil.reservationQ);
return request;
}
+
+ @Test(timeout = 3, expected = ApplicationNotFoundException.class)
+ public void testShouldNotRetryForeverForNonNetworkExceptions() throws
Exception {
+YarnConfiguration conf = new YarnConfiguration();
+conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, -1);
+
+ResourceManager rm = null;
+YarnClient yarnClient = null;
+try {
+ // start rm
+ rm = new ResourceManager();
+ rm.init(conf);
+ rm.start();
+
+ yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(conf);
+ yarnClient.start();
+
+ // create invalid application id
+ ApplicationId appId = ApplicationId.newInstance(1430126768L, 10645);
+
+ // RM should throw ApplicationNotFoundException exception
+ yarnClient.getApplicationReport(appId);
+} finally {
+ if (yarnClient != null) {
+yarnClient.stop();
+ }
+ if (rm != null) {
+rm.stop();
+ }
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b68c338b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
index fa8d642..28628f3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
@@ -224,19 +224,20 @@ public class RMProxyT {
failoverSleepBaseMs, failoverSleepMaxMs);
}
-if (waitForEver) {
- return RetryPolicies.RETRY_FOREVER;
-}
-
if (rmConnectionRetryIntervalMS 0) {
throw new YarnRuntimeException(Invalid Configuration. +
YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
should not be negative.);
}
-RetryPolicy retryPolicy
hadoop git commit: YARN-3646. Applications are getting stuck some times in case of retry policy forever. Contributed by Raju Bairishetti.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 6d7e7ef1c - 157ecb224
YARN-3646. Applications are getting stuck some times in case of retry
policy forever. Contributed by Raju Bairishetti.
(cherry picked from commit 0305316d6932e6f1a05021354d77b6934e57e171)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/157ecb22
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/157ecb22
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/157ecb22
Branch: refs/heads/branch-2
Commit: 157ecb22418e22abf7078d0d0eb2e2249b8941b8
Parents: 6d7e7ef
Author: Devaraj K deva...@apache.org
Authored: Thu May 21 20:14:44 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu May 21 20:16:09 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../yarn/client/api/impl/TestYarnClient.java| 32
.../org/apache/hadoop/yarn/client/RMProxy.java | 15 -
3 files changed, 43 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/157ecb22/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2939373..12cf925 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -496,6 +496,9 @@ Release 2.7.1 - UNRELEASED
YARN-3694. Fix dead link for TimelineServer REST API.
(Jagadesh Kiran N via aajisaka)
+YARN-3646. Applications are getting stuck some times in case of retry
+policy forever. (Raju Bairishetti via devaraj)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/157ecb22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 511fa4a..bc40b9a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -1265,4 +1265,36 @@ public class TestYarnClient {
ReservationSystemTestUtil.reservationQ);
return request;
}
+
+ @Test(timeout = 3, expected = ApplicationNotFoundException.class)
+ public void testShouldNotRetryForeverForNonNetworkExceptions() throws
Exception {
+YarnConfiguration conf = new YarnConfiguration();
+conf.setInt(YarnConfiguration.RESOURCEMANAGER_CONNECT_MAX_WAIT_MS, -1);
+
+ResourceManager rm = null;
+YarnClient yarnClient = null;
+try {
+ // start rm
+ rm = new ResourceManager();
+ rm.init(conf);
+ rm.start();
+
+ yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(conf);
+ yarnClient.start();
+
+ // create invalid application id
+ ApplicationId appId = ApplicationId.newInstance(1430126768L, 10645);
+
+ // RM should throw ApplicationNotFoundException exception
+ yarnClient.getApplicationReport(appId);
+} finally {
+ if (yarnClient != null) {
+yarnClient.stop();
+ }
+ if (rm != null) {
+rm.stop();
+ }
+}
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/157ecb22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
index fa8d642..28628f3 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/RMProxy.java
@@ -224,19 +224,20 @@ public class RMProxyT {
failoverSleepBaseMs, failoverSleepMaxMs);
}
-if (waitForEver) {
- return RetryPolicies.RETRY_FOREVER;
-}
-
if (rmConnectionRetryIntervalMS 0) {
throw new YarnRuntimeException(Invalid Configuration. +
YarnConfiguration.RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS +
should not be negative.);
}
-RetryPolicy retryPolicy
hadoop git commit: MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists to avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d9455c790 - b9865cbf3
MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists
to avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED
state. Contributed by zhihai xu.
(cherry picked from commit 1ea90144d58443a7431ff33ba49ee19278ebe42b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b9865cbf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b9865cbf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b9865cbf
Branch: refs/heads/branch-2
Commit: b9865cbf3d435903835afa23c74c5e2ecfbba1bc
Parents: d9455c7
Author: Devaraj K deva...@apache.org
Authored: Fri May 15 12:31:32 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri May 15 12:34:08 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 4 +++
.../mapreduce/v2/hs/HistoryFileManager.java | 11 +++---
.../mapreduce/v2/hs/TestHistoryFileManager.java | 35
3 files changed, 46 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9865cbf/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index fd4cd72..9d943bd 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -167,6 +167,10 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-5708. Duplicate String.format in
YarnOutputFiles.getSpillFileForWrite.
(Konstantin Weitz via devaraj)
+MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists
to
+avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state.
+(zhihai xu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9865cbf/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 6420303..0b9eaee 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -303,8 +303,9 @@ public class HistoryFileManager extends AbstractService {
private JobIndexInfo jobIndexInfo;
private HistoryInfoState state;
-private HistoryFileInfo(Path historyFile, Path confFile, Path summaryFile,
-JobIndexInfo jobIndexInfo, boolean isInDone) {
+@VisibleForTesting
+protected HistoryFileInfo(Path historyFile, Path confFile,
+Path summaryFile, JobIndexInfo jobIndexInfo, boolean isInDone) {
this.historyFile = historyFile;
this.confFile = confFile;
this.summaryFile = summaryFile;
@@ -337,7 +338,8 @@ public class HistoryFileManager extends AbstractService {
+ historyFile = + historyFile;
}
-private synchronized void moveToDone() throws IOException {
+@VisibleForTesting
+synchronized void moveToDone() throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug(moveToDone: + historyFile);
}
@@ -368,7 +370,8 @@ public class HistoryFileManager extends AbstractService {
paths.add(confFile);
}
-if (summaryFile == null) {
+if (summaryFile == null || !intermediateDoneDirFc.util().exists(
+summaryFile)) {
LOG.info(No summary file for job: + jobId);
} else {
String jobSummaryString = getJobSummary(intermediateDoneDirFc,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b9865cbf/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index 1c5cc5c..7694de9 100644
---
a/hadoop-mapreduce-project/hadoop
hadoop git commit: MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists to avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/trunk 3bef7c80a - 1ea90144d
MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists
to avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED
state. Contributed by zhihai xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1ea90144
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1ea90144
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1ea90144
Branch: refs/heads/trunk
Commit: 1ea90144d58443a7431ff33ba49ee19278ebe42b
Parents: 3bef7c8
Author: Devaraj K deva...@apache.org
Authored: Fri May 15 12:31:32 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri May 15 12:31:32 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 4 +++
.../mapreduce/v2/hs/HistoryFileManager.java | 11 +++---
.../mapreduce/v2/hs/TestHistoryFileManager.java | 35
3 files changed, 46 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea90144/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index bc1f427..7e34297 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -425,6 +425,10 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-5708. Duplicate String.format in
YarnOutputFiles.getSpillFileForWrite.
(Konstantin Weitz via devaraj)
+MAPREDUCE-6273. HistoryFileManager should check whether summaryFile exists
to
+avoid FileNotFoundException causing HistoryFileInfo into MOVE_FAILED state.
+(zhihai xu via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea90144/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 69f814d..f0786da 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -303,8 +303,9 @@ public class HistoryFileManager extends AbstractService {
private JobIndexInfo jobIndexInfo;
private HistoryInfoState state;
-private HistoryFileInfo(Path historyFile, Path confFile, Path summaryFile,
-JobIndexInfo jobIndexInfo, boolean isInDone) {
+@VisibleForTesting
+protected HistoryFileInfo(Path historyFile, Path confFile,
+Path summaryFile, JobIndexInfo jobIndexInfo, boolean isInDone) {
this.historyFile = historyFile;
this.confFile = confFile;
this.summaryFile = summaryFile;
@@ -337,7 +338,8 @@ public class HistoryFileManager extends AbstractService {
+ historyFile = + historyFile;
}
-private synchronized void moveToDone() throws IOException {
+@VisibleForTesting
+synchronized void moveToDone() throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug(moveToDone: + historyFile);
}
@@ -368,7 +370,8 @@ public class HistoryFileManager extends AbstractService {
paths.add(confFile);
}
-if (summaryFile == null) {
+if (summaryFile == null || !intermediateDoneDirFc.util().exists(
+summaryFile)) {
LOG.info(No summary file for job: + jobId);
} else {
String jobSummaryString = getJobSummary(intermediateDoneDirFc,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1ea90144/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index 1c5cc5c..7694de9 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop
hadoop git commit: MAPREDUCE-5708. Duplicate String.format in YarnOutputFiles.getSpillFileForWrite. Contributed by Konstantin Weitz.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 4bbcffa51 - 454236ec1
MAPREDUCE-5708. Duplicate String.format in
YarnOutputFiles.getSpillFileForWrite. Contributed by Konstantin Weitz.
(cherry picked from commit 05ff54c66c49301c4ec2549704d9d459e784572c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/454236ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/454236ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/454236ec
Branch: refs/heads/branch-2
Commit: 454236ec19c29e9366882f501f67169b51424b3d
Parents: 4bbcffa
Author: Devaraj K deva...@apache.org
Authored: Thu May 14 22:09:54 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu May 14 22:11:15 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/454236ec/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 2885c0e..fd4cd72 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -164,6 +164,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6366. mapreduce.terasort.final.sync configuration in TeraSort
doesn't work. (Takuya Fukudome via ozawa)
+MAPREDUCE-5708. Duplicate String.format in
YarnOutputFiles.getSpillFileForWrite.
+(Konstantin Weitz via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/454236ec/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
index e08e093..e099b8f 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
@@ -157,8 +157,8 @@ public class YarnOutputFiles extends MapOutputFile {
public Path getSpillFileForWrite(int spillNumber, long size)
throws IOException {
return lDirAlloc.getLocalPathForWrite(
-String.format(String.format(SPILL_FILE_PATTERN,
-conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber)), size, conf);
+String.format(SPILL_FILE_PATTERN,
+conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber), size, conf);
}
/**
hadoop git commit: MAPREDUCE-5708. Duplicate String.format in YarnOutputFiles.getSpillFileForWrite. Contributed by Konstantin Weitz.
Repository: hadoop
Updated Branches:
refs/heads/trunk def9136e0 - 05ff54c66
MAPREDUCE-5708. Duplicate String.format in
YarnOutputFiles.getSpillFileForWrite. Contributed by Konstantin Weitz.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/05ff54c6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/05ff54c6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/05ff54c6
Branch: refs/heads/trunk
Commit: 05ff54c66c49301c4ec2549704d9d459e784572c
Parents: def9136
Author: Devaraj K deva...@apache.org
Authored: Thu May 14 22:09:54 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu May 14 22:09:54 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05ff54c6/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 7fe8483..bc1f427 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -422,6 +422,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6366. mapreduce.terasort.final.sync configuration in TeraSort
doesn't work. (Takuya Fukudome via ozawa)
+MAPREDUCE-5708. Duplicate String.format in
YarnOutputFiles.getSpillFileForWrite.
+(Konstantin Weitz via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05ff54c6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
index e08e093..e099b8f 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/YarnOutputFiles.java
@@ -157,8 +157,8 @@ public class YarnOutputFiles extends MapOutputFile {
public Path getSpillFileForWrite(int spillNumber, long size)
throws IOException {
return lDirAlloc.getLocalPathForWrite(
-String.format(String.format(SPILL_FILE_PATTERN,
-conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber)), size, conf);
+String.format(SPILL_FILE_PATTERN,
+conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber), size, conf);
}
/**
hadoop git commit: YARN-3629. NodeID is always printed as null in node manager initialization log. Contributed by nijel.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 fb5b0ebb4 - 5fceebd69
YARN-3629. NodeID is always printed as null in node manager
initialization log. Contributed by nijel.
(cherry picked from commit 5c2f05cd9bad9bf9beb0f4ca18f4ae1bc3e84499)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5fceebd6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5fceebd6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5fceebd6
Branch: refs/heads/branch-2
Commit: 5fceebd69698bf22e840ca9f527aaa3e63375a9c
Parents: fb5b0eb
Author: Devaraj K deva...@apache.org
Authored: Tue May 12 22:20:25 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Tue May 12 22:22:40 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fceebd6/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 69acb61..d5d57a7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -349,6 +349,9 @@ Release 2.8.0 - UNRELEASED
YARN-3602.
TestResourceLocalizationService.testPublicResourceInitializesLocalDir
fails Intermittently due to IOException from cleanup. (zhihai xu via xgong)
+YARN-3629. NodeID is always printed as null in node manager
initialization log.
+(nijel via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5fceebd6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index b1ab5f1..0eb7ff4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -199,7 +199,7 @@ public class NodeStatusUpdaterImpl extends AbstractService
implements
+ durationToTrackStoppedContainers);
}
super.serviceInit(conf);
-LOG.info(Initialized nodemanager for + nodeId + : +
+LOG.info(Initialized nodemanager with : +
physical-memory= + memoryMb + virtual-memory= + virtualMemoryMb +
virtual-cores= + virtualCores);
@@ -213,6 +213,7 @@ public class NodeStatusUpdaterImpl extends AbstractService
implements
// NodeManager is the last service to start, so NodeId is available.
this.nodeId = this.context.getNodeId();
+LOG.info(Node ID assigned is : + this.nodeId);
this.httpPort = this.context.getHttpPort();
this.nodeManagerVersionId = YarnVersionInfo.getVersion();
try {
hadoop git commit: YARN-3629. NodeID is always printed as null in node manager initialization log. Contributed by nijel.
Repository: hadoop
Updated Branches:
refs/heads/trunk f4e2b3cc0 - 5c2f05cd9
YARN-3629. NodeID is always printed as null in node manager
initialization log. Contributed by nijel.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5c2f05cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5c2f05cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5c2f05cd
Branch: refs/heads/trunk
Commit: 5c2f05cd9bad9bf9beb0f4ca18f4ae1bc3e84499
Parents: f4e2b3c
Author: Devaraj K deva...@apache.org
Authored: Tue May 12 22:20:25 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Tue May 12 22:20:25 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c2f05cd/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d5fc259..131161f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -394,6 +394,9 @@ Release 2.8.0 - UNRELEASED
YARN-3602.
TestResourceLocalizationService.testPublicResourceInitializesLocalDir
fails Intermittently due to IOException from cleanup. (zhihai xu via xgong)
+YARN-3629. NodeID is always printed as null in node manager
initialization log.
+(nijel via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5c2f05cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index b1ab5f1..0eb7ff4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -199,7 +199,7 @@ public class NodeStatusUpdaterImpl extends AbstractService
implements
+ durationToTrackStoppedContainers);
}
super.serviceInit(conf);
-LOG.info(Initialized nodemanager for + nodeId + : +
+LOG.info(Initialized nodemanager with : +
physical-memory= + memoryMb + virtual-memory= + virtualMemoryMb +
virtual-cores= + virtualCores);
@@ -213,6 +213,7 @@ public class NodeStatusUpdaterImpl extends AbstractService
implements
// NodeManager is the last service to start, so NodeId is available.
this.nodeId = this.context.getNodeId();
+LOG.info(Node ID assigned is : + this.nodeId);
this.httpPort = this.context.getHttpPort();
this.nodeManagerVersionId = YarnVersionInfo.getVersion();
try {
hadoop git commit: YARN-3513. Remove unused variables in ContainersMonitorImpl and add debug log for overall resource usage by all containers. Contributed by Naganarasimha G R.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 cda196299 - f887243f8
YARN-3513. Remove unused variables in ContainersMonitorImpl and add debug
log for overall resource usage by all containers. Contributed by
Naganarasimha G R.
(cherry picked from commit 8badd82ce256e4dc8c234961120d62a88358ab39)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f887243f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f887243f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f887243f
Branch: refs/heads/branch-2
Commit: f887243f884b137c398bc05f75f457f76c029f16
Parents: cda1962
Author: Devaraj K deva...@apache.org
Authored: Tue May 12 16:54:38 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Tue May 12 16:56:26 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../monitor/ContainersMonitorImpl.java | 27 ++--
2 files changed, 22 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f887243f/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 815d35c..69acb61 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -182,6 +182,9 @@ Release 2.8.0 - UNRELEASED
YARN-3587. Fix the javadoc of DelegationTokenSecretManager in yarn, etc.
projects. (Gabor Liptak via junping_du)
+YARN-3513. Remove unused variables in ContainersMonitorImpl and add debug
+log for overall resource usage by all containers. (Naganarasimha G R via
devaraj)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f887243f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 76bbda1..d1e5e01 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -389,8 +389,10 @@ public class ContainersMonitorImpl extends AbstractService
implements
// Now do the monitoring for the trackingContainers
// Check memory usage and kill any overflowing containers
-long vmemStillInUsage = 0;
-long pmemStillInUsage = 0;
+long vmemUsageByAllContainers = 0;
+long pmemByAllContainers = 0;
+long cpuUsagePercentPerCoreByAllContainers = 0;
+long cpuUsageTotalCoresByAllContainers = 0;
for (IteratorMap.EntryContainerId, ProcessTreeInfo it =
trackingContainers.entrySet().iterator(); it.hasNext();) {
@@ -504,6 +506,13 @@ public class ContainersMonitorImpl extends AbstractService
implements
containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_PMEM;
}
+// Accounting the total memory in usage for all containers
+vmemUsageByAllContainers += currentVmemUsage;
+pmemByAllContainers += currentPmemUsage;
+// Accounting the total cpu usage for all containers
+cpuUsagePercentPerCoreByAllContainers += cpuUsagePercentPerCore;
+cpuUsageTotalCoresByAllContainers += cpuUsagePercentPerCore;
+
if (isMemoryOverLimit) {
// Virtual or physical memory over limit. Fail the container and
// remove
@@ -520,12 +529,6 @@ public class ContainersMonitorImpl extends AbstractService
implements
containerExitStatus, msg));
it.remove();
LOG.info(Removed ProcessTree with root + pId);
-} else {
- // Accounting the total memory in usage for all containers that
- // are still
- // alive and within limits.
- vmemStillInUsage += currentVmemUsage;
- pmemStillInUsage += currentPmemUsage
hadoop git commit: YARN-3513. Remove unused variables in ContainersMonitorImpl and add debug log for overall resource usage by all containers. Contributed by Naganarasimha G R.
Repository: hadoop
Updated Branches:
refs/heads/trunk 360dff590 - 8badd82ce
YARN-3513. Remove unused variables in ContainersMonitorImpl and add debug
log for overall resource usage by all containers. Contributed by
Naganarasimha G R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8badd82c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8badd82c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8badd82c
Branch: refs/heads/trunk
Commit: 8badd82ce256e4dc8c234961120d62a88358ab39
Parents: 360dff5
Author: Devaraj K deva...@apache.org
Authored: Tue May 12 16:54:38 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Tue May 12 16:54:38 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../monitor/ContainersMonitorImpl.java | 27 ++--
2 files changed, 22 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8badd82c/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2412dce..d5fc259 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -227,6 +227,9 @@ Release 2.8.0 - UNRELEASED
YARN-3587. Fix the javadoc of DelegationTokenSecretManager in yarn, etc.
projects. (Gabor Liptak via junping_du)
+YARN-3513. Remove unused variables in ContainersMonitorImpl and add debug
+log for overall resource usage by all containers. (Naganarasimha G R via
devaraj)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8badd82c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 76bbda1..d1e5e01 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -389,8 +389,10 @@ public class ContainersMonitorImpl extends AbstractService
implements
// Now do the monitoring for the trackingContainers
// Check memory usage and kill any overflowing containers
-long vmemStillInUsage = 0;
-long pmemStillInUsage = 0;
+long vmemUsageByAllContainers = 0;
+long pmemByAllContainers = 0;
+long cpuUsagePercentPerCoreByAllContainers = 0;
+long cpuUsageTotalCoresByAllContainers = 0;
for (IteratorMap.EntryContainerId, ProcessTreeInfo it =
trackingContainers.entrySet().iterator(); it.hasNext();) {
@@ -504,6 +506,13 @@ public class ContainersMonitorImpl extends AbstractService
implements
containerExitStatus = ContainerExitStatus.KILLED_EXCEEDED_PMEM;
}
+// Accounting the total memory in usage for all containers
+vmemUsageByAllContainers += currentVmemUsage;
+pmemByAllContainers += currentPmemUsage;
+// Accounting the total cpu usage for all containers
+cpuUsagePercentPerCoreByAllContainers += cpuUsagePercentPerCore;
+cpuUsageTotalCoresByAllContainers += cpuUsagePercentPerCore;
+
if (isMemoryOverLimit) {
// Virtual or physical memory over limit. Fail the container and
// remove
@@ -520,12 +529,6 @@ public class ContainersMonitorImpl extends AbstractService
implements
containerExitStatus, msg));
it.remove();
LOG.info(Removed ProcessTree with root + pId);
-} else {
- // Accounting the total memory in usage for all containers that
- // are still
- // alive and within limits.
- vmemStillInUsage += currentVmemUsage;
- pmemStillInUsage += currentPmemUsage;
}
} catch (Exception e) {
// Log the exception and proceed to the next
hadoop git commit: YARN-3358. Audit log not present while refreshing Service ACLs. Contributed by Varun Saxena.
Repository: hadoop Updated Branches: refs/heads/trunk e5e492a96 - ef3d66d46 YARN-3358. Audit log not present while refreshing Service ACLs. Contributed by Varun Saxena. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef3d66d4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef3d66d4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef3d66d4 Branch: refs/heads/trunk Commit: ef3d66d4624d360e75c016e36824a6782d6a9746 Parents: e5e492a Author: Devaraj K deva...@apache.org Authored: Fri May 8 12:14:10 2015 +0530 Committer: Devaraj K deva...@apache.org Committed: Fri May 8 12:14:10 2015 +0530 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../apache/hadoop/yarn/server/resourcemanager/AdminService.java | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3d66d4/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a698a7d..489cee3 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -400,6 +400,9 @@ Release 2.7.1 - UNRELEASED YARN-3385. Fixed a race-condition in ResourceManager's ZooKeeper based state-store to avoid crashing on duplicate deletes. (Zhihai Xu via vinodkv) +YARN-3358. Audit log not present while refreshing Service ACLs. +(Varun Saxena via devaraj) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef3d66d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 0ad90c0..1ee8b3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -508,7 +508,9 @@ public class AdminService extends CompositeService implements conf, policyProvider); rmContext.getResourceTrackerService().refreshServiceAcls( conf, policyProvider); - + +RMAuditLogger.logSuccess(user.getShortUserName(), argName, AdminService); + return recordFactory.newRecordInstance(RefreshServiceAclsResponse.class); }
hadoop git commit: YARN-3358. Audit log not present while refreshing Service ACLs. Contributed by Varun Saxena.
Repository: hadoop Updated Branches: refs/heads/branch-2 089d420a8 - 0d286d104 YARN-3358. Audit log not present while refreshing Service ACLs. Contributed by Varun Saxena. (cherry picked from commit ef3d66d4624d360e75c016e36824a6782d6a9746) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d286d10 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d286d10 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d286d10 Branch: refs/heads/branch-2 Commit: 0d286d1041ed6bec02bb7dbe4aa54165c259c7c3 Parents: 089d420 Author: Devaraj K deva...@apache.org Authored: Fri May 8 12:14:10 2015 +0530 Committer: Devaraj K deva...@apache.org Committed: Fri May 8 12:15:45 2015 +0530 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../apache/hadoop/yarn/server/resourcemanager/AdminService.java | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d286d10/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0523f4f..063d2bd 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -358,6 +358,9 @@ Release 2.7.1 - UNRELEASED YARN-3385. Fixed a race-condition in ResourceManager's ZooKeeper based state-store to avoid crashing on duplicate deletes. (Zhihai Xu via vinodkv) +YARN-3358. Audit log not present while refreshing Service ACLs. +(Varun Saxena via devaraj) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d286d10/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 0ad90c0..1ee8b3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -508,7 +508,9 @@ public class AdminService extends CompositeService implements conf, policyProvider); rmContext.getResourceTrackerService().refreshServiceAcls( conf, policyProvider); - + +RMAuditLogger.logSuccess(user.getShortUserName(), argName, AdminService); + return recordFactory.newRecordInstance(RefreshServiceAclsResponse.class); }
hadoop git commit: YARN-3358. Audit log not present while refreshing Service ACLs. Contributed by Varun Saxena.
Repository: hadoop Updated Branches: refs/heads/branch-2.7 74bd9666d - ada2f74b1 YARN-3358. Audit log not present while refreshing Service ACLs. Contributed by Varun Saxena. (cherry picked from commit ef3d66d4624d360e75c016e36824a6782d6a9746) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ada2f74b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ada2f74b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ada2f74b Branch: refs/heads/branch-2.7 Commit: ada2f74b1774ba9e0c7158fb36b277ebf94e7460 Parents: 74bd966 Author: Devaraj K deva...@apache.org Authored: Fri May 8 12:14:10 2015 +0530 Committer: Devaraj K deva...@apache.org Committed: Fri May 8 12:16:13 2015 +0530 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../apache/hadoop/yarn/server/resourcemanager/AdminService.java | 4 +++- 2 files changed, 6 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada2f74b/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 0229c29..82d4fa6 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -65,6 +65,9 @@ Release 2.7.1 - UNRELEASED YARN-3385. Fixed a race-condition in ResourceManager's ZooKeeper based state-store to avoid crashing on duplicate deletes. (Zhihai Xu via vinodkv) +YARN-3358. Audit log not present while refreshing Service ACLs. +(Varun Saxena via devaraj) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/ada2f74b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java index 6180995..35aba7e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java @@ -488,7 +488,9 @@ public class AdminService extends CompositeService implements conf, policyProvider); rmContext.getResourceTrackerService().refreshServiceAcls( conf, policyProvider); - + +RMAuditLogger.logSuccess(user.getShortUserName(), argName, AdminService); + return recordFactory.newRecordInstance(RefreshServiceAclsResponse.class); }
hadoop git commit: MAPREDUCE-6342. Make POM project names consistent. Contributed by Rohith.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 eaa5f3e44 - ade5165ec
MAPREDUCE-6342. Make POM project names consistent. Contributed by Rohith.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ade5165e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ade5165e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ade5165e
Branch: refs/heads/branch-2
Commit: ade5165ecf8e19159175d3edf7035c1b1a585430
Parents: eaa5f3e
Author: Devaraj K deva...@apache.org
Authored: Fri May 8 15:20:35 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri May 8 15:20:35 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 2 ++
.../hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml| 2 +-
.../hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml | 2 +-
.../hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml | 2 +-
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 2 +-
.../hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml | 2 +-
.../hadoop-mapreduce-client-jobclient/pom.xml | 2 +-
.../hadoop-mapreduce-client-shuffle/pom.xml| 2 +-
hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 2 +-
hadoop-mapreduce-project/pom.xml | 2 +-
10 files changed, 11 insertions(+), 9 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ade5165e/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index bcbd229..e139d38 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -134,6 +134,8 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6356. Misspelling of threshold in log4j.properties for tests.
(Brahma Reddy Battula via aajisaka)
+MAPREDUCE-6342. Make POM project names consistent. (Rohith via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ade5165e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index 5341672e..0a5343a 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -25,7 +25,7 @@
groupIdorg.apache.hadoop/groupId
artifactIdhadoop-mapreduce-client-app/artifactId
version2.8.0-SNAPSHOT/version
- namehadoop-mapreduce-client-app/name
+ nameApache Hadoop MapReduce App/name
properties
applink.base${project.build.directory}/${project.name}/applink.base
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ade5165e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
index 824e753..d561270 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
@@ -25,7 +25,7 @@
groupIdorg.apache.hadoop/groupId
artifactIdhadoop-mapreduce-client-common/artifactId
version2.8.0-SNAPSHOT/version
- namehadoop-mapreduce-client-common/name
+ nameApache Hadoop MapReduce Common/name
properties
!-- Needed for generating FindBugs warnings using parent pom --
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ade5165e/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index 44d1c17..a8efa93 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -25,7 +25,7 @@
groupIdorg.apache.hadoop/groupId
artifactIdhadoop-mapreduce-client-core/artifactId
version2.8.0-SNAPSHOT/version
- namehadoop-mapreduce-client-core/name
+ nameApache Hadoop
hadoop git commit: MAPREDUCE-6342. Make POM project names consistent. Contributed by Rohith.
Repository: hadoop
Updated Branches:
refs/heads/trunk 4c6816faf - 2401ff76b
MAPREDUCE-6342. Make POM project names consistent. Contributed by Rohith.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2401ff76
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2401ff76
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2401ff76
Branch: refs/heads/trunk
Commit: 2401ff76bea603b2c4e59338d5954ef60d6c4005
Parents: 4c6816f
Author: Devaraj K deva...@apache.org
Authored: Fri May 8 15:17:52 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri May 8 15:17:52 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 2 ++
.../hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml| 2 +-
.../hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml | 2 +-
.../hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml | 2 +-
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 2 +-
.../hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml | 2 +-
.../hadoop-mapreduce-client-jobclient/pom.xml | 2 +-
.../hadoop-mapreduce-client-nativetask/pom.xml | 2 +-
.../hadoop-mapreduce-client-shuffle/pom.xml| 2 +-
hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 2 +-
hadoop-mapreduce-project/pom.xml | 2 +-
11 files changed, 12 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2401ff76/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 265b5b1..f534001 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -386,6 +386,8 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6356. Misspelling of threshold in log4j.properties for tests.
(Brahma Reddy Battula via aajisaka)
+MAPREDUCE-6342. Make POM project names consistent. (Rohith via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2401ff76/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index 4f6f837..29fd570 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -25,7 +25,7 @@
groupIdorg.apache.hadoop/groupId
artifactIdhadoop-mapreduce-client-app/artifactId
version3.0.0-SNAPSHOT/version
- namehadoop-mapreduce-client-app/name
+ nameApache Hadoop MapReduce App/name
properties
applink.base${project.build.directory}/${project.name}/applink.base
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2401ff76/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
index ab00baa..82b71de 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
@@ -25,7 +25,7 @@
groupIdorg.apache.hadoop/groupId
artifactIdhadoop-mapreduce-client-common/artifactId
version3.0.0-SNAPSHOT/version
- namehadoop-mapreduce-client-common/name
+ nameApache Hadoop MapReduce Common/name
properties
!-- Needed for generating FindBugs warnings using parent pom --
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2401ff76/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index 2a4a174..19ce44b 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -25,7 +25,7 @@
groupIdorg.apache.hadoop/groupId
artifactIdhadoop-mapreduce-client-core/artifactId
version3.0.0-SNAPSHOT
hadoop git commit: YARN-2784. Make POM project names consistent. Contributed by Rohith.
Repository: hadoop Updated Branches: refs/heads/trunk 9757864fd - 241a72af0 YARN-2784. Make POM project names consistent. Contributed by Rohith. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/241a72af Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/241a72af Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/241a72af Branch: refs/heads/trunk Commit: 241a72af0dd19040be333d77749f8be17b8aafc7 Parents: 9757864 Author: Devaraj K deva...@apache.org Authored: Fri May 8 19:23:53 2015 +0530 Committer: Devaraj K deva...@apache.org Committed: Fri May 8 19:23:53 2015 +0530 -- hadoop-yarn-project/CHANGES.txt| 2 ++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml| 2 +- .../hadoop-yarn-applications-distributedshell/pom.xml | 2 +- .../hadoop-yarn-applications-unmanaged-am-launcher/pom.xml | 2 +- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml | 2 +- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml | 2 +- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml | 2 +- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml | 2 +- .../hadoop-yarn-server-applicationhistoryservice/pom.xml | 2 +- .../hadoop-yarn-server/hadoop-yarn-server-common/pom.xml | 2 +- .../hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml | 2 +- .../hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml | 2 +- .../hadoop-yarn-server-sharedcachemanager/pom.xml | 2 +- .../hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml| 2 +- .../hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml| 2 +- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml | 2 +- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml | 2 +- hadoop-yarn-project/hadoop-yarn/pom.xml| 2 +- hadoop-yarn-project/pom.xml| 2 +- 19 files changed, 20 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/241a72af/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ee6049b..26febe9 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -191,6 +191,8 @@ Release 2.8.0 - UNRELEASED YARN-3169. Drop YARN's overview document. (Brahma Reddy Battula via ozawa) +YARN-2784. Make POM project names consistent. (Rohith via devaraj) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/241a72af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml index a763d39..dc9c469 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml @@ -25,7 +25,7 @@ groupIdorg.apache.hadoop/groupId artifactIdhadoop-yarn-api/artifactId version3.0.0-SNAPSHOT/version - namehadoop-yarn-api/name + nameApache Hadoop YARN API/name properties !-- Needed for generating FindBugs warnings using parent pom -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/241a72af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml index d91c67b..5b4440f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml @@ -25,7 +25,7 @@ groupIdorg.apache.hadoop/groupId artifactIdhadoop-yarn-applications-distributedshell/artifactId version3.0.0-SNAPSHOT/version - namehadoop-yarn-applications-distributedshell/name + nameApache Hadoop YARN DistributedShell/name properties !-- Needed for generating FindBugs warnings using parent pom -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/241a72af/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
hadoop git commit: YARN-2784. Make POM project names consistent. Contributed by Rohith.
Repository: hadoop Updated Branches: refs/heads/branch-2 bcb863b15 - bf8be5629 YARN-2784. Make POM project names consistent. Contributed by Rohith. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bf8be562 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bf8be562 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bf8be562 Branch: refs/heads/branch-2 Commit: bf8be562926d0fa5a557d037bf9c934fa86a096c Parents: bcb863b Author: Devaraj K deva...@apache.org Authored: Fri May 8 19:26:11 2015 +0530 Committer: Devaraj K deva...@apache.org Committed: Fri May 8 19:26:11 2015 +0530 -- hadoop-yarn-project/CHANGES.txt| 2 ++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml| 2 +- .../hadoop-yarn-applications-distributedshell/pom.xml | 2 +- .../hadoop-yarn-applications-unmanaged-am-launcher/pom.xml | 2 +- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml | 2 +- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml | 2 +- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml | 2 +- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml | 2 +- .../hadoop-yarn-server-applicationhistoryservice/pom.xml | 2 +- .../hadoop-yarn-server/hadoop-yarn-server-common/pom.xml | 2 +- .../hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml | 2 +- .../hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml | 2 +- .../hadoop-yarn-server-sharedcachemanager/pom.xml | 2 +- .../hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml| 2 +- .../hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml| 2 +- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml | 2 +- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml | 2 +- hadoop-yarn-project/hadoop-yarn/pom.xml| 2 +- hadoop-yarn-project/pom.xml| 2 +- 19 files changed, 20 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf8be562/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 594ec71..32c5ae1 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -146,6 +146,8 @@ Release 2.8.0 - UNRELEASED YARN-3169. Drop YARN's overview document. (Brahma Reddy Battula via ozawa) +YARN-2784. Make POM project names consistent. (Rohith via devaraj) + OPTIMIZATIONS YARN-3339. TestDockerContainerExecutor should pull a single image and not http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf8be562/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml index 3a3f5ea..a872930 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml @@ -25,7 +25,7 @@ groupIdorg.apache.hadoop/groupId artifactIdhadoop-yarn-api/artifactId version2.8.0-SNAPSHOT/version - namehadoop-yarn-api/name + nameApache Hadoop YARN API/name properties !-- Needed for generating FindBugs warnings using parent pom -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf8be562/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml index 87ca0a0..24f8bcc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml @@ -25,7 +25,7 @@ groupIdorg.apache.hadoop/groupId artifactIdhadoop-yarn-applications-distributedshell/artifactId version2.8.0-SNAPSHOT/version - namehadoop-yarn-applications-distributedshell/name + nameApache Hadoop YARN DistributedShell/name properties !-- Needed for generating FindBugs warnings using parent pom -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf8be562/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
hadoop git commit: MAPREDUCE-5981. Log levels of certain MR logs can be changed to DEBUG. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/trunk 6f622672b - dc2b2ae31
MAPREDUCE-5981. Log levels of certain MR logs can be changed to DEBUG.
Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc2b2ae3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc2b2ae3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc2b2ae3
Branch: refs/heads/trunk
Commit: dc2b2ae31f2eb6dae324c2e14ed7660ce605a89b
Parents: 6f62267
Author: Devaraj K deva...@apache.org
Authored: Fri May 8 21:27:54 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri May 8 21:27:54 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java| 2 +-
.../hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java | 4 ++--
3 files changed, 6 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc2b2ae3/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index f534001..c97a7f6 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -316,6 +316,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6079. Rename JobImpl#username to reporterUserName.
(Tsuyoshi Ozawa via aajisaka)
+MAPREDUCE-5981. Log levels of certain MR logs can be changed to DEBUG.
+(Varun Saxena via devaraj)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc2b2ae3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
index 4b80dc9..1e03387 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
@@ -448,7 +448,7 @@ class FetcherK,V extends Thread {
LOG.debug(url=+msgToEncode+;encHash=+encHash+;replyHash=+replyHash);
// verify that replyHash is HMac of encHash
SecureShuffleUtils.verifyReply(replyHash, encHash, shuffleSecretKey);
-LOG.info(for url=+msgToEncode+ sent hash and received reply);
+LOG.debug(for url=+msgToEncode+ sent hash and received reply);
}
private void setupShuffleConnection(String encHash) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc2b2ae3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
index 37f4af3..8317672 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
@@ -413,7 +413,7 @@ public class ShuffleSchedulerImplK,V implements
ShuffleSchedulerK,V {
pendingHosts.remove(host);
host.markBusy();
- LOG.info(Assigning + host + with + host.getNumKnownMapOutputs() +
+ LOG.debug(Assigning + host + with + host.getNumKnownMapOutputs() +
to + Thread.currentThread().getName());
shuffleStart.set(Time.monotonicNow());
@@ -443,7 +443,7 @@ public class ShuffleSchedulerImplK,V implements
ShuffleSchedulerK,V {
host.addKnownMap(id);
}
}
-LOG.info(assigned + includedMaps + of + totalSize + to +
+LOG.debug(assigned + includedMaps + of + totalSize + to +
host + to + Thread.currentThread().getName());
return result;
}
hadoop git commit: MAPREDUCE-5981. Log levels of certain MR logs can be changed to DEBUG. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 802a5775f - a957baeb7
MAPREDUCE-5981. Log levels of certain MR logs can be changed to DEBUG.
Contributed by Varun Saxena.
(cherry picked from commit dc2b2ae31f2eb6dae324c2e14ed7660ce605a89b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a957baeb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a957baeb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a957baeb
Branch: refs/heads/branch-2
Commit: a957baeb71eb75aa259902d9d6dce2a64f5a3e38
Parents: 802a577
Author: Devaraj K deva...@apache.org
Authored: Fri May 8 21:27:54 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri May 8 21:28:52 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java| 2 +-
.../hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java | 4 ++--
3 files changed, 6 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a957baeb/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index e139d38..ac8c3be 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -64,6 +64,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6079. Rename JobImpl#username to reporterUserName.
(Tsuyoshi Ozawa via aajisaka)
+MAPREDUCE-5981. Log levels of certain MR logs can be changed to DEBUG.
+(Varun Saxena via devaraj)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a957baeb/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
index 4b80dc9..1e03387 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/Fetcher.java
@@ -448,7 +448,7 @@ class FetcherK,V extends Thread {
LOG.debug(url=+msgToEncode+;encHash=+encHash+;replyHash=+replyHash);
// verify that replyHash is HMac of encHash
SecureShuffleUtils.verifyReply(replyHash, encHash, shuffleSecretKey);
-LOG.info(for url=+msgToEncode+ sent hash and received reply);
+LOG.debug(for url=+msgToEncode+ sent hash and received reply);
}
private void setupShuffleConnection(String encHash) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a957baeb/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
index 37f4af3..8317672 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/ShuffleSchedulerImpl.java
@@ -413,7 +413,7 @@ public class ShuffleSchedulerImplK,V implements
ShuffleSchedulerK,V {
pendingHosts.remove(host);
host.markBusy();
- LOG.info(Assigning + host + with + host.getNumKnownMapOutputs() +
+ LOG.debug(Assigning + host + with + host.getNumKnownMapOutputs() +
to + Thread.currentThread().getName());
shuffleStart.set(Time.monotonicNow());
@@ -443,7 +443,7 @@ public class ShuffleSchedulerImplK,V implements
ShuffleSchedulerK,V {
host.addKnownMap(id);
}
}
-LOG.info(assigned + includedMaps + of + totalSize + to +
+LOG.debug(assigned + includedMaps + of + totalSize + to +
host + to + Thread.currentThread().getName());
return result;
}
hadoop git commit: YARN-3592. Fix typos in RMNodeLabelsManager. Contributed by Sunil G.
Repository: hadoop
Updated Branches:
refs/heads/trunk 66988476d - a28cd02ba
YARN-3592. Fix typos in RMNodeLabelsManager. Contributed by Sunil G.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a28cd02b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a28cd02b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a28cd02b
Branch: refs/heads/trunk
Commit: a28cd02ba026733e409807f3ea1b36f7d57bc273
Parents: 6698847
Author: Devaraj K deva...@apache.org
Authored: Fri May 8 10:54:36 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri May 8 10:54:36 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 2 ++
.../resourcemanager/nodelabels/RMNodeLabelsManager.java | 10 +-
2 files changed, 7 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a28cd02b/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f8a2cc0..a698a7d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -342,6 +342,8 @@ Release 2.8.0 - UNRELEASED
YARN-3572. Correct typos in WritingYarnApplications.md.
(Gabor Liptak via aajisaka)
+YARN-3592. Fix typos in RMNodeLabelsManager. (Sunil G via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a28cd02b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
index 25e5bc09..696b99b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
@@ -46,11 +46,11 @@ import com.google.common.collect.ImmutableSet;
public class RMNodeLabelsManager extends CommonNodeLabelsManager {
protected static class Queue {
-protected SetString acccessibleNodeLabels;
+protected SetString accessibleNodeLabels;
protected Resource resource;
protected Queue() {
- acccessibleNodeLabels =
+ accessibleNodeLabels =
Collections.newSetFromMap(new ConcurrentHashMapString, Boolean());
resource = Resource.newInstance(0, 0);
}
@@ -98,7 +98,7 @@ public class RMNodeLabelsManager extends
CommonNodeLabelsManager {
// check if any queue contains this label
for (EntryString, Queue entry : queueCollections.entrySet()) {
String queueName = entry.getKey();
-SetString queueLabels = entry.getValue().acccessibleNodeLabels;
+SetString queueLabels = entry.getValue().accessibleNodeLabels;
if (queueLabels.contains(label)) {
throw new IOException(Cannot remove label= + label
+ , because queue= + queueName + is using this label.
@@ -275,7 +275,7 @@ public class RMNodeLabelsManager extends
CommonNodeLabelsManager {
continue;
}
-q.acccessibleNodeLabels.addAll(labels);
+q.accessibleNodeLabels.addAll(labels);
for (Host host : nodeCollections.values()) {
for (EntryNodeId, Node nentry : host.nms.entrySet()) {
NodeId nodeId = nentry.getKey();
@@ -468,7 +468,7 @@ public class RMNodeLabelsManager extends
CommonNodeLabelsManager {
}
for (String label : nodeLabels) {
- if (q.acccessibleNodeLabels.contains(label)) {
+ if (q.accessibleNodeLabels.contains(label)) {
return true;
}
}
hadoop git commit: YARN-3592. Fix typos in RMNodeLabelsManager. Contributed by Sunil G.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 d0fff5d32 - dce2381dc
YARN-3592. Fix typos in RMNodeLabelsManager. Contributed by Sunil G.
(cherry picked from commit a28cd02ba026733e409807f3ea1b36f7d57bc273)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dce2381d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dce2381d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dce2381d
Branch: refs/heads/branch-2
Commit: dce2381dc4a877fcbfb869f115152ecd44a92173
Parents: d0fff5d
Author: Devaraj K deva...@apache.org
Authored: Fri May 8 10:54:36 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Fri May 8 10:55:53 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 2 ++
.../resourcemanager/nodelabels/RMNodeLabelsManager.java | 10 +-
2 files changed, 7 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dce2381d/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b49940d..0523f4f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -297,6 +297,8 @@ Release 2.8.0 - UNRELEASED
YARN-3572. Correct typos in WritingYarnApplications.md.
(Gabor Liptak via aajisaka)
+YARN-3592. Fix typos in RMNodeLabelsManager. (Sunil G via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dce2381d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
index 25e5bc09..696b99b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
@@ -46,11 +46,11 @@ import com.google.common.collect.ImmutableSet;
public class RMNodeLabelsManager extends CommonNodeLabelsManager {
protected static class Queue {
-protected SetString acccessibleNodeLabels;
+protected SetString accessibleNodeLabels;
protected Resource resource;
protected Queue() {
- acccessibleNodeLabels =
+ accessibleNodeLabels =
Collections.newSetFromMap(new ConcurrentHashMapString, Boolean());
resource = Resource.newInstance(0, 0);
}
@@ -98,7 +98,7 @@ public class RMNodeLabelsManager extends
CommonNodeLabelsManager {
// check if any queue contains this label
for (EntryString, Queue entry : queueCollections.entrySet()) {
String queueName = entry.getKey();
-SetString queueLabels = entry.getValue().acccessibleNodeLabels;
+SetString queueLabels = entry.getValue().accessibleNodeLabels;
if (queueLabels.contains(label)) {
throw new IOException(Cannot remove label= + label
+ , because queue= + queueName + is using this label.
@@ -275,7 +275,7 @@ public class RMNodeLabelsManager extends
CommonNodeLabelsManager {
continue;
}
-q.acccessibleNodeLabels.addAll(labels);
+q.accessibleNodeLabels.addAll(labels);
for (Host host : nodeCollections.values()) {
for (EntryNodeId, Node nentry : host.nms.entrySet()) {
NodeId nodeId = nentry.getKey();
@@ -468,7 +468,7 @@ public class RMNodeLabelsManager extends
CommonNodeLabelsManager {
}
for (String label : nodeLabels) {
- if (q.acccessibleNodeLabels.contains(label)) {
+ if (q.accessibleNodeLabels.contains(label)) {
return true;
}
}
hadoop git commit: MAPREDUCE-6339. Job history file is not flushed correctly because isTimerActive flag is not set true when flushTimerTask is scheduled. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/trunk aa2245044 - f5b38477f
MAPREDUCE-6339. Job history file is not flushed correctly because
isTimerActive flag is not set true when flushTimerTask is scheduled.
Contributed by zhihai xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5b38477
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5b38477
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5b38477
Branch: refs/heads/trunk
Commit: f5b38477f9d0827b238fadd260c1dd2889531fd4
Parents: aa22450
Author: Devaraj K deva...@apache.org
Authored: Thu Apr 30 12:33:22 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Apr 30 12:33:22 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java | 7 +++
.../mapreduce/jobhistory/TestJobHistoryEventHandler.java | 2 ++
3 files changed, 12 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5b38477/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index e9e1a4b..c073470 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -382,6 +382,9 @@ Release 2.7.1 - UNRELEASED
MAPREDUCE-6334. Fetcher#copyMapOutput is leaking usedMemory upon
IOException during InMemoryMapOutput shuffle handler (Eric Payne via jlowe)
+MAPREDUCE-6339. Job history file is not flushed correctly because
isTimerActive
+flag is not set true when flushTimerTask is scheduled. (zhihai xu via
devaraj)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5b38477/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index a0e7041..6b0ea79 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -72,6 +72,7 @@ import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.node.ArrayNode;
import org.codehaus.jackson.node.ObjectNode;
+import com.google.common.annotations.VisibleForTesting;
/**
* The job history events get routed to this class. This class writes the Job
* history events to the DFS directly into a staging dir and then moved to a
@@ -1259,6 +1260,7 @@ public class JobHistoryEventHandler extends
AbstractService
if (!isTimerShutDown) {
flushTimerTask = new FlushTimerTask(this);
flushTimer.schedule(flushTimerTask, flushTimeout);
+isTimerActive = true;
}
}
}
@@ -1378,4 +1380,9 @@ public class JobHistoryEventHandler extends
AbstractService
}
return JobState.KILLED.toString();
}
+
+ @VisibleForTesting
+ boolean getFlushTimerStatus() {
+return isTimerActive;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5b38477/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index de260c9..49be35b 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -223,10 +223,12 @@ public class TestJobHistoryEventHandler {
}
handleNextNEvents(jheh, 9);
+ Assert.assertTrue
hadoop git commit: MAPREDUCE-6339. Job history file is not flushed correctly because isTimerActive flag is not set true when flushTimerTask is scheduled. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 9f8412ac6 - c3d083a9f
MAPREDUCE-6339. Job history file is not flushed correctly because
isTimerActive flag is not set true when flushTimerTask is scheduled.
Contributed by zhihai xu.
(cherry picked from commit f5b38477f9d0827b238fadd260c1dd2889531fd4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c3d083a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c3d083a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c3d083a9
Branch: refs/heads/branch-2
Commit: c3d083a9ff692408cae26098a0f1eea9de6a02c3
Parents: 9f8412a
Author: Devaraj K deva...@apache.org
Authored: Thu Apr 30 12:33:22 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Apr 30 12:36:00 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java | 7 +++
.../mapreduce/jobhistory/TestJobHistoryEventHandler.java | 2 ++
3 files changed, 12 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3d083a9/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index fae743b..9e31a30 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -130,6 +130,9 @@ Release 2.7.1 - UNRELEASED
MAPREDUCE-6334. Fetcher#copyMapOutput is leaking usedMemory upon
IOException during InMemoryMapOutput shuffle handler (Eric Payne via jlowe)
+MAPREDUCE-6339. Job history file is not flushed correctly because
isTimerActive
+flag is not set true when flushTimerTask is scheduled. (zhihai xu via
devaraj)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3d083a9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index a0e7041..6b0ea79 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -72,6 +72,7 @@ import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.node.ArrayNode;
import org.codehaus.jackson.node.ObjectNode;
+import com.google.common.annotations.VisibleForTesting;
/**
* The job history events get routed to this class. This class writes the Job
* history events to the DFS directly into a staging dir and then moved to a
@@ -1259,6 +1260,7 @@ public class JobHistoryEventHandler extends
AbstractService
if (!isTimerShutDown) {
flushTimerTask = new FlushTimerTask(this);
flushTimer.schedule(flushTimerTask, flushTimeout);
+isTimerActive = true;
}
}
}
@@ -1378,4 +1380,9 @@ public class JobHistoryEventHandler extends
AbstractService
}
return JobState.KILLED.toString();
}
+
+ @VisibleForTesting
+ boolean getFlushTimerStatus() {
+return isTimerActive;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3d083a9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index de260c9..49be35b 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -223,10 +223,12 @@ public class TestJobHistoryEventHandler
hadoop git commit: MAPREDUCE-6339. Job history file is not flushed correctly because isTimerActive flag is not set true when flushTimerTask is scheduled. Contributed by zhihai xu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 6ac2b5712 - 9828638fa
MAPREDUCE-6339. Job history file is not flushed correctly because
isTimerActive flag is not set true when flushTimerTask is scheduled.
Contributed by zhihai xu.
(cherry picked from commit f5b38477f9d0827b238fadd260c1dd2889531fd4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9828638f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9828638f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9828638f
Branch: refs/heads/branch-2.7
Commit: 9828638fac850ff45bf8a80dd29f0254bc61f35e
Parents: 6ac2b57
Author: Devaraj K deva...@apache.org
Authored: Thu Apr 30 12:33:22 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Apr 30 12:37:36 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java | 7 +++
.../mapreduce/jobhistory/TestJobHistoryEventHandler.java | 2 ++
3 files changed, 12 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9828638f/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 10491f9..11475d6 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -26,6 +26,9 @@ Release 2.7.1 - UNRELEASED
MAPREDUCE-6334. Fetcher#copyMapOutput is leaking usedMemory upon
IOException during InMemoryMapOutput shuffle handler (Eric Payne via jlowe)
+MAPREDUCE-6339. Job history file is not flushed correctly because
isTimerActive
+flag is not set true when flushTimerTask is scheduled. (zhihai xu via
devaraj)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9828638f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index a0e7041..6b0ea79 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -72,6 +72,7 @@ import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.node.ArrayNode;
import org.codehaus.jackson.node.ObjectNode;
+import com.google.common.annotations.VisibleForTesting;
/**
* The job history events get routed to this class. This class writes the Job
* history events to the DFS directly into a staging dir and then moved to a
@@ -1259,6 +1260,7 @@ public class JobHistoryEventHandler extends
AbstractService
if (!isTimerShutDown) {
flushTimerTask = new FlushTimerTask(this);
flushTimer.schedule(flushTimerTask, flushTimeout);
+isTimerActive = true;
}
}
}
@@ -1378,4 +1380,9 @@ public class JobHistoryEventHandler extends
AbstractService
}
return JobState.KILLED.toString();
}
+
+ @VisibleForTesting
+ boolean getFlushTimerStatus() {
+return isTimerActive;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9828638f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index 43e3dbe..7ed0b63 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -223,10 +223,12 @@ public class TestJobHistoryEventHandler
hadoop git commit: Moving MAPREDUCE-6252 to the 2.7.1 CHANGES.txt
Repository: hadoop Updated Branches: refs/heads/branch-2 a82addd61 - 1d03ac353 Moving MAPREDUCE-6252 to the 2.7.1 CHANGES.txt (cherry picked from commit 99fe03e439b0f9afd01754d998c6eb64f0f70300) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d03ac35 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d03ac35 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d03ac35 Branch: refs/heads/branch-2 Commit: 1d03ac35335b4efe63212bf3846ccbdee95fa96c Parents: a82addd Author: Devaraj K deva...@apache.org Authored: Tue Apr 28 12:42:16 2015 +0530 Committer: Devaraj K deva...@apache.org Committed: Tue Apr 28 12:44:07 2015 +0530 -- hadoop-mapreduce-project/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d03ac35/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index cbafabd..7439f4c 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -101,9 +101,6 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6333. TestEvents,TestAMWebServicesTasks,TestAppController are broken due to MAPREDUCE-6297. (Siqi Li via gera) -MAPREDUCE-6252. JobHistoryServer should not fail when encountering a -missing directory. (Craig Welch via devaraj) - MAPREDUCE-6341. Fix typo in mapreduce tutorial. (John Michael Luy via ozawa) @@ -127,6 +124,9 @@ Release 2.7.1 - UNRELEASED MAPREDUCE-6324. Fixed MapReduce uber jobs to not fail the udpate of AM-RM tokens when they roll-over. (Jason Lowe via vinodkv) +MAPREDUCE-6252. JobHistoryServer should not fail when encountering a +missing directory. (Craig Welch via devaraj) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES
hadoop git commit: Moving MAPREDUCE-6252 to the 2.7.1 CHANGES.txt
Repository: hadoop Updated Branches: refs/heads/trunk feb68cb54 - 99fe03e43 Moving MAPREDUCE-6252 to the 2.7.1 CHANGES.txt Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99fe03e4 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99fe03e4 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99fe03e4 Branch: refs/heads/trunk Commit: 99fe03e439b0f9afd01754d998c6eb64f0f70300 Parents: feb68cb Author: Devaraj K deva...@apache.org Authored: Tue Apr 28 12:42:16 2015 +0530 Committer: Devaraj K deva...@apache.org Committed: Tue Apr 28 12:42:16 2015 +0530 -- hadoop-mapreduce-project/CHANGES.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/99fe03e4/hadoop-mapreduce-project/CHANGES.txt -- diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt index eb6feb9..d27a022 100644 --- a/hadoop-mapreduce-project/CHANGES.txt +++ b/hadoop-mapreduce-project/CHANGES.txt @@ -350,9 +350,6 @@ Release 2.8.0 - UNRELEASED MAPREDUCE-6333. TestEvents,TestAMWebServicesTasks,TestAppController are broken due to MAPREDUCE-6297. (Siqi Li via gera) -MAPREDUCE-6252. JobHistoryServer should not fail when encountering a -missing directory. (Craig Welch via devaraj) - MAPREDUCE-6341. Fix typo in mapreduce tutorial. (John Michael Luy via ozawa) @@ -376,6 +373,9 @@ Release 2.7.1 - UNRELEASED MAPREDUCE-6324. Fixed MapReduce uber jobs to not fail the udpate of AM-RM tokens when they roll-over. (Jason Lowe via vinodkv) +MAPREDUCE-6252. JobHistoryServer should not fail when encountering a +missing directory. (Craig Welch via devaraj) + Release 2.7.0 - 2015-04-20 INCOMPATIBLE CHANGES
hadoop git commit: MAPREDUCE-6252. JobHistoryServer should not fail when encountering a missing directory. Contributed by Craig Welch.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.7 3a7bfdf31 - a940f1922
MAPREDUCE-6252. JobHistoryServer should not fail when encountering a
missing directory. Contributed by Craig Welch.
(cherry picked from commit 5e67c4d384193b38a85655c8f93193596821faa5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a940f192
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a940f192
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a940f192
Branch: refs/heads/branch-2.7
Commit: a940f192268441afb47010e30fbdb8853756d639
Parents: 3a7bfdf
Author: Devaraj K deva...@apache.org
Authored: Mon Apr 27 15:01:42 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Tue Apr 28 12:31:46 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +++
.../mapreduce/v2/hs/HistoryFileManager.java | 19 ++---
.../mapreduce/v2/hs/TestHistoryFileManager.java | 22
3 files changed, 37 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a940f192/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 2af375d..85c6fea 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -20,6 +20,9 @@ Release 2.7.1 - UNRELEASED
MAPREDUCE-6324. Fixed MapReduce uber jobs to not fail the udpate of AM-RM
tokens when they roll-over. (Jason Lowe via vinodkv)
+MAPREDUCE-6252. JobHistoryServer should not fail when encountering a
+missing directory. (Craig Welch via devaraj)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a940f192/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 6b9f146..6420303 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -740,17 +740,22 @@ public class HistoryFileManager extends AbstractService {
}
}
- private static ListFileStatus scanDirectory(Path path, FileContext fc,
+ @VisibleForTesting
+ protected static ListFileStatus scanDirectory(Path path, FileContext fc,
PathFilter pathFilter) throws IOException {
path = fc.makeQualified(path);
ListFileStatus jhStatusList = new ArrayListFileStatus();
-RemoteIteratorFileStatus fileStatusIter = fc.listStatus(path);
-while (fileStatusIter.hasNext()) {
- FileStatus fileStatus = fileStatusIter.next();
- Path filePath = fileStatus.getPath();
- if (fileStatus.isFile() pathFilter.accept(filePath)) {
-jhStatusList.add(fileStatus);
+try {
+ RemoteIteratorFileStatus fileStatusIter = fc.listStatus(path);
+ while (fileStatusIter.hasNext()) {
+FileStatus fileStatus = fileStatusIter.next();
+Path filePath = fileStatus.getPath();
+if (fileStatus.isFile() pathFilter.accept(filePath)) {
+ jhStatusList.add(fileStatus);
+}
}
+} catch (FileNotFoundException fe) {
+ LOG.error(Error while scanning directory + path, fe);
}
return jhStatusList;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a940f192/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index e2e943a..1c5cc5c 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop
hadoop git commit: MAPREDUCE-6252. JobHistoryServer should not fail when encountering a missing directory. Contributed by Craig Welch.
Repository: hadoop
Updated Branches:
refs/heads/trunk 618ba707f - 5e67c4d38
MAPREDUCE-6252. JobHistoryServer should not fail when encountering a
missing directory. Contributed by Craig Welch.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e67c4d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e67c4d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e67c4d3
Branch: refs/heads/trunk
Commit: 5e67c4d384193b38a85655c8f93193596821faa5
Parents: 618ba70
Author: Devaraj K deva...@apache.org
Authored: Mon Apr 27 15:01:42 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Apr 27 15:01:42 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +++
.../mapreduce/v2/hs/HistoryFileManager.java | 19 ++---
.../mapreduce/v2/hs/TestHistoryFileManager.java | 22
3 files changed, 37 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e67c4d3/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 4166e6a..f9488fb 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -349,6 +349,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6333. TestEvents,TestAMWebServicesTasks,TestAppController are
broken due to MAPREDUCE-6297. (Siqi Li via gera)
+MAPREDUCE-6252. JobHistoryServer should not fail when encountering a
+missing directory. (Craig Welch via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e67c4d3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 65f8a4f..69f814d 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -740,17 +740,22 @@ public class HistoryFileManager extends AbstractService {
}
}
- private static ListFileStatus scanDirectory(Path path, FileContext fc,
+ @VisibleForTesting
+ protected static ListFileStatus scanDirectory(Path path, FileContext fc,
PathFilter pathFilter) throws IOException {
path = fc.makeQualified(path);
ListFileStatus jhStatusList = new ArrayListFileStatus();
-RemoteIteratorFileStatus fileStatusIter = fc.listStatus(path);
-while (fileStatusIter.hasNext()) {
- FileStatus fileStatus = fileStatusIter.next();
- Path filePath = fileStatus.getPath();
- if (fileStatus.isFile() pathFilter.accept(filePath)) {
-jhStatusList.add(fileStatus);
+try {
+ RemoteIteratorFileStatus fileStatusIter = fc.listStatus(path);
+ while (fileStatusIter.hasNext()) {
+FileStatus fileStatus = fileStatusIter.next();
+Path filePath = fileStatus.getPath();
+if (fileStatus.isFile() pathFilter.accept(filePath)) {
+ jhStatusList.add(fileStatus);
+}
}
+} catch (FileNotFoundException fe) {
+ LOG.error(Error while scanning directory + path, fe);
}
return jhStatusList;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e67c4d3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index e2e943a..1c5cc5c 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
@@ -21,13 +21,17 @@ package
hadoop git commit: MAPREDUCE-6252. JobHistoryServer should not fail when encountering a missing directory. Contributed by Craig Welch.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b73cfd7eb - 84ae26573
MAPREDUCE-6252. JobHistoryServer should not fail when encountering a
missing directory. Contributed by Craig Welch.
(cherry picked from commit 5e67c4d384193b38a85655c8f93193596821faa5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84ae2657
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84ae2657
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84ae2657
Branch: refs/heads/branch-2
Commit: 84ae26573c4fccbaab042b7f639bf1d5a1960281
Parents: b73cfd7
Author: Devaraj K deva...@apache.org
Authored: Mon Apr 27 15:01:42 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Apr 27 15:02:44 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +++
.../mapreduce/v2/hs/HistoryFileManager.java | 19 ++---
.../mapreduce/v2/hs/TestHistoryFileManager.java | 22
3 files changed, 37 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84ae2657/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index a1d3523..bdbe3c5 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -101,6 +101,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6333. TestEvents,TestAMWebServicesTasks,TestAppController are
broken due to MAPREDUCE-6297. (Siqi Li via gera)
+MAPREDUCE-6252. JobHistoryServer should not fail when encountering a
+missing directory. (Craig Welch via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84ae2657/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 6b9f146..6420303 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -740,17 +740,22 @@ public class HistoryFileManager extends AbstractService {
}
}
- private static ListFileStatus scanDirectory(Path path, FileContext fc,
+ @VisibleForTesting
+ protected static ListFileStatus scanDirectory(Path path, FileContext fc,
PathFilter pathFilter) throws IOException {
path = fc.makeQualified(path);
ListFileStatus jhStatusList = new ArrayListFileStatus();
-RemoteIteratorFileStatus fileStatusIter = fc.listStatus(path);
-while (fileStatusIter.hasNext()) {
- FileStatus fileStatus = fileStatusIter.next();
- Path filePath = fileStatus.getPath();
- if (fileStatus.isFile() pathFilter.accept(filePath)) {
-jhStatusList.add(fileStatus);
+try {
+ RemoteIteratorFileStatus fileStatusIter = fc.listStatus(path);
+ while (fileStatusIter.hasNext()) {
+FileStatus fileStatus = fileStatusIter.next();
+Path filePath = fileStatus.getPath();
+if (fileStatus.isFile() pathFilter.accept(filePath)) {
+ jhStatusList.add(fileStatus);
+}
}
+} catch (FileNotFoundException fe) {
+ LOG.error(Error while scanning directory + path, fe);
}
return jhStatusList;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84ae2657/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index e2e943a..1c5cc5c 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2
hadoop git commit: MAPREDUCE-6242. Progress report log is incredibly excessive in application master. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 503d8e416 - 943d9ee60
MAPREDUCE-6242. Progress report log is incredibly excessive in application
master. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/943d9ee6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/943d9ee6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/943d9ee6
Branch: refs/heads/branch-2
Commit: 943d9ee603b28a769805f94b83ea90ccba717813
Parents: 503d8e4
Author: Devaraj K deva...@apache.org
Authored: Mon Mar 23 22:48:00 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Mar 23 22:48:00 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../java/org/apache/hadoop/mapred/Task.java | 15 +-
.../apache/hadoop/mapreduce/MRJobConfig.java| 5 +
.../hadoop/mapred/TestTaskProgressReporter.java | 147 +++
4 files changed, 166 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/943d9ee6/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index e399d3e..06aba95 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -52,6 +52,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6281. Fix javadoc in Terasort. (Albert Chu via ozawa)
+MAPREDUCE-6242. Progress report log is incredibly excessive in
+application master. (Varun Saxena via devaraj)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/943d9ee6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 1ea1666..9fab545 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -70,6 +70,8 @@ import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* Base class for tasks.
*/
@@ -227,6 +229,11 @@ abstract public class Task implements Writable,
Configurable {
gcUpdater = new GcTimeUpdater();
}
+ @VisibleForTesting
+ void setTaskDone() {
+taskDone.set(true);
+ }
+
// Accessors
@@ -534,9 +541,6 @@ abstract public class Task implements Writable,
Configurable {
public abstract void run(JobConf job, TaskUmbilicalProtocol umbilical)
throws IOException, ClassNotFoundException, InterruptedException;
- /** The number of milliseconds between progress reports. */
- public static final int PROGRESS_INTERVAL = 3000;
-
private transient Progress taskProgress = new Progress();
// Current counters
@@ -711,6 +715,9 @@ abstract public class Task implements Writable,
Configurable {
int remainingRetries = MAX_RETRIES;
// get current flag value and reset it as well
boolean sendProgress = resetProgressFlag();
+ long taskProgressInterval =
+ conf.getLong(MRJobConfig.TASK_PROGRESS_REPORT_INTERVAL,
+ MRJobConfig.DEFAULT_TASK_PROGRESS_REPORT_INTERVAL);
while (!taskDone.get()) {
synchronized (lock) {
done = false;
@@ -722,7 +729,7 @@ abstract public class Task implements Writable,
Configurable {
if (taskDone.get()) {
break;
}
-lock.wait(PROGRESS_INTERVAL);
+lock.wait(taskProgressInterval);
}
if (taskDone.get()) {
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/943d9ee6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org
hadoop git commit: MAPREDUCE-6242. Progress report log is incredibly excessive in application master. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/trunk 36af4a913 - 7e6f384dd
MAPREDUCE-6242. Progress report log is incredibly excessive in application
master. Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e6f384d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e6f384d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e6f384d
Branch: refs/heads/trunk
Commit: 7e6f384dd742de21f29e96ee76df5316529c9019
Parents: 36af4a9
Author: Devaraj K deva...@apache.org
Authored: Mon Mar 23 22:51:20 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Mon Mar 23 22:51:20 2015 +0530
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../java/org/apache/hadoop/mapred/Task.java | 13 +-
.../apache/hadoop/mapreduce/MRJobConfig.java| 5 +
.../hadoop/mapred/TestTaskProgressReporter.java | 160 +++
4 files changed, 177 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e6f384d/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 20505b6..b8a2a1c 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -300,6 +300,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6281. Fix javadoc in Terasort. (Albert Chu via ozawa)
+MAPREDUCE-6242. Progress report log is incredibly excessive in
+application master. (Varun Saxena via devaraj)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e6f384d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 7fa5d02..bf5ca22 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -229,6 +229,11 @@ abstract public class Task implements Writable,
Configurable {
gcUpdater = new GcTimeUpdater();
}
+ @VisibleForTesting
+ void setTaskDone() {
+taskDone.set(true);
+ }
+
// Accessors
@@ -536,9 +541,6 @@ abstract public class Task implements Writable,
Configurable {
public abstract void run(JobConf job, TaskUmbilicalProtocol umbilical)
throws IOException, ClassNotFoundException, InterruptedException;
- /** The number of milliseconds between progress reports. */
- public static final int PROGRESS_INTERVAL = 3000;
-
private transient Progress taskProgress = new Progress();
// Current counters
@@ -714,6 +716,9 @@ abstract public class Task implements Writable,
Configurable {
int remainingRetries = MAX_RETRIES;
// get current flag value and reset it as well
boolean sendProgress = resetProgressFlag();
+ long taskProgressInterval =
+ conf.getLong(MRJobConfig.TASK_PROGRESS_REPORT_INTERVAL,
+ MRJobConfig.DEFAULT_TASK_PROGRESS_REPORT_INTERVAL);
while (!taskDone.get()) {
synchronized (lock) {
done = false;
@@ -726,7 +731,7 @@ abstract public class Task implements Writable,
Configurable {
if (taskDone.get()) {
break;
}
-lock.wait(PROGRESS_INTERVAL);
+lock.wait(taskProgressInterval);
}
if (taskDone.get()) {
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e6f384d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index f0a6ddf..947c814 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++
b/hadoop-mapreduce-project/hadoop
hadoop git commit: YARN-3357. Move TestFifoScheduler to FIFO package. Contributed by Rohith Sharmaks.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 25515d142 - a888dac34
YARN-3357. Move TestFifoScheduler to FIFO package. Contributed by Rohith
Sharmaks.
(cherry picked from commit 93d0f4acc8454b7a460da8883fbd14f14a575370)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a888dac3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a888dac3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a888dac3
Branch: refs/heads/branch-2
Commit: a888dac3441c46f82539d9192f265a54d909d593
Parents: 25515d1
Author: Devaraj K deva...@apache.org
Authored: Thu Mar 19 12:16:52 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Mar 19 12:18:08 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../TestApplicationMasterService.java | 3 +-
.../resourcemanager/TestFifoScheduler.java | 629 ---
.../capacity/TestContainerAllocation.java | 4 +-
.../scheduler/fifo/TestFifoScheduler.java | 598 --
5 files changed, 564 insertions(+), 673 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a888dac3/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cc26923..66079e2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -14,6 +14,9 @@ Release 2.8.0 - UNRELEASED
YARN-3273. Improve scheduler UI to facilitate scheduling analysis and
debugging. (Rohith Sharmaks via jianhe)
+YARN-3357. Move TestFifoScheduler to FIFO package. (Rohith Sharmaks
+via devaraj)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a888dac3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index d1f0ede..ca5c7a4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -55,7 +55,8 @@ import java.util.Map;
import static java.lang.Thread.sleep;
public class TestApplicationMasterService {
- private static final Log LOG = LogFactory.getLog(TestFifoScheduler.class);
+ private static final Log LOG = LogFactory
+ .getLog(TestApplicationMasterService.class);
private final int GB = 1024;
private static YarnConfiguration conf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a888dac3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
deleted file mode 100644
index b74faeb..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
+++ /dev/null
@@ -1,629 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* License); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an AS IS BASIS,
-* WITHOUT
hadoop git commit: YARN-3357. Move TestFifoScheduler to FIFO package. Contributed by Rohith Sharmaks.
Repository: hadoop
Updated Branches:
refs/heads/trunk 355e008ff - 93d0f4acc
YARN-3357. Move TestFifoScheduler to FIFO package. Contributed by Rohith
Sharmaks.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/93d0f4ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/93d0f4ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/93d0f4ac
Branch: refs/heads/trunk
Commit: 93d0f4acc8454b7a460da8883fbd14f14a575370
Parents: 355e008
Author: Devaraj K deva...@apache.org
Authored: Thu Mar 19 12:16:52 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Mar 19 12:16:52 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../TestApplicationMasterService.java | 3 +-
.../resourcemanager/TestFifoScheduler.java | 629 ---
.../capacity/TestContainerAllocation.java | 4 +-
.../scheduler/fifo/TestFifoScheduler.java | 598 --
5 files changed, 564 insertions(+), 673 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93d0f4ac/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7322c6b..0a143dd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -62,6 +62,9 @@ Release 2.8.0 - UNRELEASED
YARN-3273. Improve scheduler UI to facilitate scheduling analysis and
debugging. (Rohith Sharmaks via jianhe)
+YARN-3357. Move TestFifoScheduler to FIFO package. (Rohith Sharmaks
+via devaraj)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93d0f4ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index d1f0ede..ca5c7a4 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -55,7 +55,8 @@ import java.util.Map;
import static java.lang.Thread.sleep;
public class TestApplicationMasterService {
- private static final Log LOG = LogFactory.getLog(TestFifoScheduler.class);
+ private static final Log LOG = LogFactory
+ .getLog(TestApplicationMasterService.class);
private final int GB = 1024;
private static YarnConfiguration conf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/93d0f4ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
deleted file mode 100644
index b74faeb..000
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
+++ /dev/null
@@ -1,629 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* License); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an AS IS BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See
hadoop git commit: YARN-3197. Confusing log generated by CapacityScheduler. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 6ddb1bc85 - 895588b43
YARN-3197. Confusing log generated by CapacityScheduler. Contributed by
Varun Saxena.
(cherry picked from commit 7179f94f9d000fc52bd9ce5aa9741aba97ec3ee8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/895588b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/895588b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/895588b4
Branch: refs/heads/branch-2
Commit: 895588b439bf6e3113a85583422484af4b89a890
Parents: 6ddb1bc
Author: Devaraj K deva...@apache.org
Authored: Tue Mar 17 15:57:57 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Tue Mar 17 15:59:19 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../resourcemanager/scheduler/capacity/CapacityScheduler.java | 5 +++--
2 files changed, 6 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/895588b4/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 538b4dd..b2f25cd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -15,6 +15,9 @@ Release 2.8.0 - UNRELEASED
BUG FIXES
+ YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena
+ via devaraj)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/895588b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 28ce264..756e537 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1279,7 +1279,8 @@ public class CapacityScheduler extends
protected synchronized void completedContainer(RMContainer rmContainer,
ContainerStatus containerStatus, RMContainerEventType event) {
if (rmContainer == null) {
- LOG.info(Null container completed...);
+ LOG.info(Container + containerStatus.getContainerId() +
+ completed with event + event);
return;
}
@@ -1291,7 +1292,7 @@ public class CapacityScheduler extends
ApplicationId appId =
container.getId().getApplicationAttemptId().getApplicationId();
if (application == null) {
- LOG.info(Container + container + of + unknown application
+ LOG.info(Container + container + of + finished application
+ appId + completed with event + event);
return;
}
hadoop git commit: YARN-3197. Confusing log generated by CapacityScheduler. Contributed by Varun Saxena.
Repository: hadoop
Updated Branches:
refs/heads/trunk 018893e81 - 7179f94f9
YARN-3197. Confusing log generated by CapacityScheduler. Contributed by
Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7179f94f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7179f94f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7179f94f
Branch: refs/heads/trunk
Commit: 7179f94f9d000fc52bd9ce5aa9741aba97ec3ee8
Parents: 018893e
Author: Devaraj K deva...@apache.org
Authored: Tue Mar 17 15:57:57 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Tue Mar 17 15:57:57 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../resourcemanager/scheduler/capacity/CapacityScheduler.java | 5 +++--
2 files changed, 6 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7179f94f/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cb68480..82934ad 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -63,6 +63,9 @@ Release 2.8.0 - UNRELEASED
BUG FIXES
+ YARN-3197. Confusing log generated by CapacityScheduler. (Varun Saxena
+ via devaraj)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7179f94f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 28ce264..756e537 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -1279,7 +1279,8 @@ public class CapacityScheduler extends
protected synchronized void completedContainer(RMContainer rmContainer,
ContainerStatus containerStatus, RMContainerEventType event) {
if (rmContainer == null) {
- LOG.info(Null container completed...);
+ LOG.info(Container + containerStatus.getContainerId() +
+ completed with event + event);
return;
}
@@ -1291,7 +1292,7 @@ public class CapacityScheduler extends
ApplicationId appId =
container.getId().getApplicationAttemptId().getApplicationId();
if (application == null) {
- LOG.info(Container + container + of + unknown application
+ LOG.info(Container + container + of + finished application
+ appId + completed with event + event);
return;
}
hadoop git commit: YARN-3256. TestClientToAMTokens#testClientTokenRace is not running against all Schedulers even when using ParameterizedSchedulerTestBase. Contributed by Anubhav Dhoot.
Repository: hadoop
Updated Branches:
refs/heads/trunk 166eecf68 - 0d4296f0e
YARN-3256. TestClientToAMTokens#testClientTokenRace is not running against
all Schedulers even when using ParameterizedSchedulerTestBase. Contributed
by Anubhav Dhoot.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0d4296f0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0d4296f0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0d4296f0
Branch: refs/heads/trunk
Commit: 0d4296f0e0f545267f2e39a868d4ffefc9844db8
Parents: 166eecf
Author: Devaraj K deva...@apache.org
Authored: Thu Feb 26 15:45:41 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Feb 26 15:45:41 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 4
.../server/resourcemanager/security/TestClientToAMTokens.java| 3 ---
2 files changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d4296f0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e5148eb..ac3cbb2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -658,6 +658,10 @@ Release 2.7.0 - UNRELEASED
YARN-3239. WebAppProxy does not support a final tracking url which has
query fragments and params (Jian He via jlowe)
+YARN-3256. TestClientToAMTokens#testClientTokenRace is not running against
+all Schedulers even when using ParameterizedSchedulerTestBase.
+(Anubhav Dhoot via devaraj)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0d4296f0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
index 78bc728..499b4d1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
@@ -33,7 +33,6 @@ import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
-import java.util.Arrays;
import java.util.Timer;
import java.util.TimerTask;
@@ -43,7 +42,6 @@ import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.Server;
@@ -421,7 +419,6 @@ public class TestClientToAMTokens extends
ParameterizedSchedulerTestBase {
@Test(timeout=2)
public void testClientTokenRace() throws Exception {
-final Configuration conf = new Configuration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
kerberos);
UserGroupInformation.setConfiguration(conf);
hadoop git commit: YARN-3256. TestClientToAMTokens#testClientTokenRace is not running against all Schedulers even when using ParameterizedSchedulerTestBase. Contributed by Anubhav Dhoot.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 f0f3d1937 - f83d2e441
YARN-3256. TestClientToAMTokens#testClientTokenRace is not running against
all Schedulers even when using ParameterizedSchedulerTestBase. Contributed
by Anubhav Dhoot.
(cherry picked from commit 0d4296f0e0f545267f2e39a868d4ffefc9844db8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f83d2e44
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f83d2e44
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f83d2e44
Branch: refs/heads/branch-2
Commit: f83d2e4410e5dadaf4c787fd1875d92e11882013
Parents: f0f3d19
Author: Devaraj K deva...@apache.org
Authored: Thu Feb 26 15:45:41 2015 +0530
Committer: Devaraj K deva...@apache.org
Committed: Thu Feb 26 15:47:32 2015 +0530
--
hadoop-yarn-project/CHANGES.txt | 4
.../server/resourcemanager/security/TestClientToAMTokens.java| 3 ---
2 files changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f83d2e44/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9046482..e399f17 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -619,6 +619,10 @@ Release 2.7.0 - UNRELEASED
YARN-3239. WebAppProxy does not support a final tracking url which has
query fragments and params (Jian He via jlowe)
+YARN-3256. TestClientToAMTokens#testClientTokenRace is not running against
+all Schedulers even when using ParameterizedSchedulerTestBase.
+(Anubhav Dhoot via devaraj)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f83d2e44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
index 78bc728..499b4d1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
@@ -33,7 +33,6 @@ import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.security.PrivilegedAction;
import java.security.PrivilegedExceptionAction;
-import java.util.Arrays;
import java.util.Timer;
import java.util.TimerTask;
@@ -43,7 +42,6 @@ import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.Server;
@@ -421,7 +419,6 @@ public class TestClientToAMTokens extends
ParameterizedSchedulerTestBase {
@Test(timeout=2)
public void testClientTokenRace() throws Exception {
-final Configuration conf = new Configuration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
kerberos);
UserGroupInformation.setConfiguration(conf);
