Author: vinodkv Date: Sun Nov 10 20:09:09 2013 New Revision: 1540535 URL: http://svn.apache.org/r1540535 Log: Forwarding YARN-321 branch to latest branch-2. svn merge ../branch-2
Modified: hadoop/common/branches/YARN-321/ (props changed) hadoop/common/branches/YARN-321/hadoop-project/ (props changed) hadoop/common/branches/YARN-321/hadoop-project-dist/pom.xml hadoop/common/branches/YARN-321/hadoop-project/pom.xml (contents, props changed) hadoop/common/branches/YARN-321/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java Propchange: hadoop/common/branches/YARN-321/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/branch-2:r1537327-1540532 Propchange: hadoop/common/branches/YARN-321/hadoop-project/ ------------------------------------------------------------------------------ Merged /hadoop/common/branches/branch-2/hadoop-project:r1537327-1540532 Modified: hadoop/common/branches/YARN-321/hadoop-project-dist/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-project-dist/pom.xml?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-project-dist/pom.xml (original) +++ hadoop/common/branches/YARN-321/hadoop-project-dist/pom.xml Sun Nov 10 20:09:09 2013 @@ -346,7 +346,7 @@ cd $${LIB_DIR} $$TAR lib* | (cd $${TARGET_DIR}/; $$UNTAR) if [ "${bundle.snappy}" = "true" ] ; then - cd ${snappy.lib} + cd "${snappy.lib}" $$TAR *snappy* | (cd $${TARGET_DIR}/; $$UNTAR) fi fi @@ -358,7 +358,7 @@ $$TAR * | (cd $${TARGET_BIN_DIR}/; $$UNTAR) if [ "${bundle.snappy.in.bin}" = "true" ] ; then if [ "${bundle.snappy}" = "true" ] ; then - cd ${snappy.lib} + cd "${snappy.lib}" $$TAR *snappy* | (cd $${TARGET_BIN_DIR}/; $$UNTAR) fi fi Modified: hadoop/common/branches/YARN-321/hadoop-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-project/pom.xml?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-project/pom.xml (original) +++ hadoop/common/branches/YARN-321/hadoop-project/pom.xml Sun Nov 10 20:09:09 2013 @@ -347,6 +347,11 @@ <version>4.2.5</version> </dependency> <dependency> + <groupId>org.apache.httpcomponents</groupId> + <artifactId>httpcore</artifactId> + <version>4.2.5</version> + </dependency> + <dependency> <groupId>commons-codec</groupId> <artifactId>commons-codec</artifactId> <version>1.4</version> @@ -553,7 +558,7 @@ <dependency> <groupId>net.java.dev.jets3t</groupId> <artifactId>jets3t</artifactId> - <version>0.6.1</version> + <version>0.9.0</version> </dependency> <dependency> <groupId>org.apache.mina</groupId> Propchange: hadoop/common/branches/YARN-321/hadoop-project/pom.xml ------------------------------------------------------------------------------ Merged /hadoop/common/branches/branch-2/hadoop-project/pom.xml:r1537327-1540532 Modified: hadoop/common/branches/YARN-321/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java (original) +++ hadoop/common/branches/YARN-321/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/Logalyzer.java Sun Nov 10 20:09:09 2013 @@ -29,6 +29,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configuration.DeprecationDelta; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.LongWritable; @@ -71,11 +72,12 @@ public class Logalyzer { "logalizer.logcomparator.column.separator"; static { - Configuration.addDeprecation("mapred.reducer.sort", - new String[] {SORT_COLUMNS}); - Configuration.addDeprecation("mapred.reducer.separator", - new String[] {COLUMN_SEPARATOR}); + Configuration.addDeprecations(new DeprecationDelta[] { + new DeprecationDelta("mapred.reducer.sort", SORT_COLUMNS), + new DeprecationDelta("mapred.reducer.separator", COLUMN_SEPARATOR) + }); } + /** A {@link Mapper} that extracts text matching a regular expression. */ public static class LogRegexMapper<K extends WritableComparable> extends MapReduceBase Modified: hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java (original) +++ hadoop/common/branches/YARN-321/hadoop-tools/hadoop-gridmix/src/main/java/org/apache/hadoop/mapred/gridmix/DistributedCacheEmulator.java Sun Nov 10 20:09:09 2013 @@ -23,6 +23,7 @@ import org.apache.commons.logging.LogFac import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configuration.DeprecationDelta; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; @@ -118,10 +119,12 @@ class DistributedCacheEmulator { { // Need to handle deprecation of these MapReduce-internal configuration // properties as MapReduce doesn't handle their deprecation. - Configuration.addDeprecation("mapred.cache.files.filesizes", - new String[] {MRJobConfig.CACHE_FILES_SIZES}); - Configuration.addDeprecation("mapred.cache.files.visibilities", - new String[] {MRJobConfig.CACHE_FILE_VISIBILITIES}); + Configuration.addDeprecations(new DeprecationDelta[] { + new DeprecationDelta("mapred.cache.files.filesizes", + MRJobConfig.CACHE_FILES_SIZES), + new DeprecationDelta("mapred.cache.files.visibilities", + MRJobConfig.CACHE_FILE_VISIBILITIES) + }); } /** Modified: hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java (original) +++ hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java Sun Nov 10 20:09:09 2013 @@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode @@ -48,7 +49,7 @@ public class NodeInfo { private String nodeAddr; private String httpAddress; private int cmdPort; - private Resource perNode; + private volatile ResourceOption perNode; private String rackName; private String healthReport; private NodeState state; @@ -56,7 +57,7 @@ public class NodeInfo { private List<ApplicationId> toCleanUpApplications; public FakeRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress, - Resource perNode, String rackName, String healthReport, + ResourceOption perNode, String rackName, String healthReport, int cmdPort, String hostName, NodeState state) { this.nodeId = nodeId; this.nodeAddr = nodeAddr; @@ -104,6 +105,10 @@ public class NodeInfo { } public Resource getTotalCapability() { + return perNode.getResource(); + } + + public ResourceOption getResourceOption() { return perNode; } @@ -153,21 +158,27 @@ public class NodeInfo { // TODO Auto-generated method stub return null; } + + @Override + public void setResourceOption(ResourceOption resourceOption) { + perNode = resourceOption; + } } public static RMNode newNodeInfo(String rackName, String hostName, - final Resource resource, int port) { + final ResourceOption resourceOption, int port) { final NodeId nodeId = newNodeID(hostName, port); final String nodeAddr = hostName + ":" + port; final String httpAddress = hostName; return new FakeRMNodeImpl(nodeId, nodeAddr, httpAddress, - resource, rackName, "Me good", + resourceOption, rackName, "Me good", port, hostName, null); } public static RMNode newNodeInfo(String rackName, String hostName, final Resource resource) { - return newNodeInfo(rackName, hostName, resource, NODE_ID++); + return newNodeInfo(rackName, hostName, ResourceOption.newInstance(resource, + RMNode.OVER_COMMIT_TIMEOUT_MILLIS_DEFAULT), NODE_ID++); } } Modified: hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java (original) +++ hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java Sun Nov 10 20:09:09 2013 @@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.NodeState; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceOption; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode @@ -138,10 +139,19 @@ public class RMNodeWrapper implements RM return updates; } -@Override -public String getNodeManagerVersion() { - // TODO Auto-generated method stub - return null; -} + @Override + public String getNodeManagerVersion() { + return node.getNodeManagerVersion(); + } + + @Override + public void setResourceOption(ResourceOption resourceOption) { + node.setResourceOption(resourceOption); + } + + @Override + public ResourceOption getResourceOption() { + return node.getResourceOption(); + } } Modified: hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java?rev=1540535&r1=1540534&r2=1540535&view=diff ============================================================================== --- hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java (original) +++ hadoop/common/branches/YARN-321/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java Sun Nov 10 20:09:09 2013 @@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configurat import org.apache.hadoop.util.ReflectionUtils; import org.apache.hadoop.yarn.api.records.ApplicationAttemptId; import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport; import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.ContainerExitStatus; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -860,5 +861,11 @@ public class ResourceSchedulerWrapper im QueueACL acl, String queueName) { return scheduler.checkAccess(callerUGI, acl, queueName); } + + @Override + public ApplicationResourceUsageReport getAppResourceUsageReport( + ApplicationAttemptId appAttemptId) { + return scheduler.getAppResourceUsageReport(appAttemptId); + } }