Author: vinodkv
Date: Sun Nov 10 20:09:09 2013
New Revision: 1540535
URL: http://svn.apache.org/r1540535
Log:
Forwarding YARN-321 branch to latest branch-2.
svn merge ../branch-2
Modified:
hadoop/common/branches/YARN-321/ (props changed)
hadoop/common/branches/YARN-321/hadoop-project/ (props changed)
hadoop/common/branches/YARN-321/hadoop-project-dist/pom.xml
hadoop/common/branches/YARN-321/hadoop-project/pom.xml (contents, props
changed)
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
Propchange: hadoop/common/branches/YARN-321/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2:r1537327-1540532
Propchange: hadoop/common/branches/YARN-321/hadoop-project/
------------------------------------------------------------------------------
Merged /hadoop/common/branches/branch-2/hadoop-project:r1537327-1540532
Modified: hadoop/common/branches/YARN-321/hadoop-project-dist/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-project-dist/pom.xml?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-project-dist/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-project-dist/pom.xml Sun Nov 10
20:09:09 2013
@@ -346,7 +346,7 @@
cd $${LIB_DIR}
$$TAR lib* | (cd $${TARGET_DIR}/; $$UNTAR)
if [ "${bundle.snappy}" = "true" ] ; then
- cd ${snappy.lib}
+ cd "${snappy.lib}"
$$TAR *snappy* | (cd $${TARGET_DIR}/; $$UNTAR)
fi
fi
@@ -358,7 +358,7 @@
$$TAR * | (cd $${TARGET_BIN_DIR}/; $$UNTAR)
if [ "${bundle.snappy.in.bin}" = "true" ] ; then
if [ "${bundle.snappy}" = "true" ] ; then
- cd ${snappy.lib}
+ cd "${snappy.lib}"
$$TAR *snappy* | (cd $${TARGET_BIN_DIR}/; $$UNTAR)
fi
fi
Modified: hadoop/common/branches/YARN-321/hadoop-project/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-project/pom.xml?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/YARN-321/hadoop-project/pom.xml Sun Nov 10 20:09:09
2013
@@ -347,6 +347,11 @@
<version>4.2.5</version>
</dependency>
<dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>4.2.5</version>
+ </dependency>
+ <dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
<version>1.4</version>
@@ -553,7 +558,7 @@
<dependency>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
- <version>0.6.1</version>
+ <version>0.9.0</version>
</dependency>
<dependency>
<groupId>org.apache.mina</groupId>
Propchange: hadoop/common/branches/YARN-321/hadoop-project/pom.xml
------------------------------------------------------------------------------
Merged
/hadoop/common/branches/branch-2/hadoop-project/pom.xml:r1537327-1540532
Modified:
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
---
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
(original)
+++
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java
Sun Nov 10 20:09:09 2013
@@ -29,6 +29,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.DeprecationDelta;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
@@ -71,11 +72,12 @@ public class Logalyzer {
"logalizer.logcomparator.column.separator";
static {
- Configuration.addDeprecation("mapred.reducer.sort",
- new String[] {SORT_COLUMNS});
- Configuration.addDeprecation("mapred.reducer.separator",
- new String[] {COLUMN_SEPARATOR});
+ Configuration.addDeprecations(new DeprecationDelta[] {
+ new DeprecationDelta("mapred.reducer.sort", SORT_COLUMNS),
+ new DeprecationDelta("mapred.reducer.separator", COLUMN_SEPARATOR)
+ });
}
+
/** A {@link Mapper} that extracts text matching a regular expression. */
public static class LogRegexMapper<K extends WritableComparable>
extends MapReduceBase
Modified:
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
---
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
(original)
+++
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java
Sun Nov 10 20:09:09 2013
@@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configuration.DeprecationDelta;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
@@ -118,10 +119,12 @@ class DistributedCacheEmulator {
{
// Need to handle deprecation of these MapReduce-internal configuration
// properties as MapReduce doesn't handle their deprecation.
- Configuration.addDeprecation("mapred.cache.files.filesizes",
- new String[] {MRJobConfig.CACHE_FILES_SIZES});
- Configuration.addDeprecation("mapred.cache.files.visibilities",
- new String[] {MRJobConfig.CACHE_FILE_VISIBILITIES});
+ Configuration.addDeprecations(new DeprecationDelta[] {
+ new DeprecationDelta("mapred.cache.files.filesizes",
+ MRJobConfig.CACHE_FILES_SIZES),
+ new DeprecationDelta("mapred.cache.files.visibilities",
+ MRJobConfig.CACHE_FILE_VISIBILITIES)
+ });
}
/**
Modified:
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
---
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
(original)
+++
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
Sun Nov 10 20:09:09 2013
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
@@ -48,7 +49,7 @@ public class NodeInfo {
private String nodeAddr;
private String httpAddress;
private int cmdPort;
- private Resource perNode;
+ private volatile ResourceOption perNode;
private String rackName;
private String healthReport;
private NodeState state;
@@ -56,7 +57,7 @@ public class NodeInfo {
private List<ApplicationId> toCleanUpApplications;
public FakeRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress,
- Resource perNode, String rackName, String healthReport,
+ ResourceOption perNode, String rackName, String healthReport,
int cmdPort, String hostName, NodeState state) {
this.nodeId = nodeId;
this.nodeAddr = nodeAddr;
@@ -104,6 +105,10 @@ public class NodeInfo {
}
public Resource getTotalCapability() {
+ return perNode.getResource();
+ }
+
+ public ResourceOption getResourceOption() {
return perNode;
}
@@ -153,21 +158,27 @@ public class NodeInfo {
// TODO Auto-generated method stub
return null;
}
+
+ @Override
+ public void setResourceOption(ResourceOption resourceOption) {
+ perNode = resourceOption;
+ }
}
public static RMNode newNodeInfo(String rackName, String hostName,
- final Resource resource, int port) {
+ final ResourceOption resourceOption, int port) {
final NodeId nodeId = newNodeID(hostName, port);
final String nodeAddr = hostName + ":" + port;
final String httpAddress = hostName;
return new FakeRMNodeImpl(nodeId, nodeAddr, httpAddress,
- resource, rackName, "Me good",
+ resourceOption, rackName, "Me good",
port, hostName, null);
}
public static RMNode newNodeInfo(String rackName, String hostName,
final Resource resource) {
- return newNodeInfo(rackName, hostName, resource, NODE_ID++);
+ return newNodeInfo(rackName, hostName, ResourceOption.newInstance(resource,
+ RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT), NODE_ID++);
}
}
Modified:
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
---
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
(original)
+++
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
Sun Nov 10 20:09:09 2013
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
@@ -138,10 +139,19 @@ public class RMNodeWrapper implements RM
return updates;
}
-@Override
-public String getNodeManagerVersion() {
- // TODO Auto-generated method stub
- return null;
-}
+ @Override
+ public String getNodeManagerVersion() {
+ return node.getNodeManagerVersion();
+ }
+
+ @Override
+ public void setResourceOption(ResourceOption resourceOption) {
+ node.setResourceOption(resourceOption);
+ }
+
+ @Override
+ public ResourceOption getResourceOption() {
+ return node.getResourceOption();
+ }
}
Modified:
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java?rev=1540535&r1=1540534&r2=1540535&view=diff
==============================================================================
---
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
(original)
+++
hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
Sun Nov 10 20:09:09 2013
@@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -860,5 +861,11 @@ public class ResourceSchedulerWrapper im
QueueACL acl, String queueName) {
return scheduler.checkAccess(callerUGI, acl, queueName);
}
+
+ @Override
+ public ApplicationResourceUsageReport getAppResourceUsageReport(
+ ApplicationAttemptId appAttemptId) {
+ return scheduler.getAppResourceUsageReport(appAttemptId);
+ }
}
