Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java?rev=1099337&r1=1099336&r2=1099337&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/NodeManagerImpl.java Wed May 4 06:53:52 2011 @@ -80,8 +80,7 @@ public class NodeManagerImpl implements /* set of containers that need to be cleaned */ private final Set<Container> containersToClean = - new TreeSet<Container>(new org.apache.hadoop.yarn.server.resourcemanager.resource.Container.ContainerComparator()); - + new TreeSet<Container>(new org.apache.hadoop.yarn.util.BuilderUtils.ContainerComparator()); /* the list of applications that have finished and need to be purged */ private final List<ApplicationId> finishedApplications = new ArrayList<ApplicationId>();
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1099337&r1=1099336&r2=1099337&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Wed May 4 06:53:52 2011 @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Application; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; +import org.apache.hadoop.yarn.util.BuilderUtils; @LimitedPrivate("yarn") @Evolving @@ -99,12 +100,12 @@ implements ResourceScheduler, CapacitySc private Resource clusterResource = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(Resource.class); private int numNodeManagers = 0; - + private Resource minimumAllocation; private Map<ApplicationId, Application> applications = new TreeMap<ApplicationId, Application>( - new org.apache.hadoop.yarn.util.BuilderUtils.ApplicationIdComparator()); + new BuilderUtils.ApplicationIdComparator()); private boolean initialized = false; Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1099337&r1=1099336&r2=1099337&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Wed May 4 06:53:52 2011 @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; +import org.apache.hadoop.yarn.util.BuilderUtils; @Private @Unstable @@ -826,25 +827,24 @@ public class LeafQueue implements Queue // crash the // scheduler. - if (availableContainers > 0) { - - List<Container> containers = - new ArrayList<Container>(); - Container container = - org.apache.hadoop.yarn.server.resourcemanager.resource.Container - .create(recordFactory, application.getApplicationId(), + if (availableContainers > 0) { + List<Container> containers = + new ArrayList<Container>(); + Container container = + BuilderUtils.newContainer(this.recordFactory, + application.getApplicationId(), application.getNewContainerId(), node.getNodeAddress(), node.getHttpAddress(), capability); - - // If security is enabled, send the container-tokens too. - if (UserGroupInformation.isSecurityEnabled()) { - ContainerToken containerToken = RecordFactoryProvider.getRecordFactory(null).newRecordInstance(ContainerToken.class); - ContainerTokenIdentifier tokenidentifier = - new ContainerTokenIdentifier(container.getId(), - container.getContainerManagerAddress(), container.getResource()); - containerToken.setIdentifier(ByteBuffer.wrap(tokenidentifier.getBytes())); - containerToken.setKind(ContainerTokenIdentifier.KIND.toString()); - containerToken.setPassword(ByteBuffer.wrap(containerTokenSecretManager + + // If security is enabled, send the container-tokens too. + if (UserGroupInformation.isSecurityEnabled()) { + ContainerToken containerToken = this.recordFactory.newRecordInstance(ContainerToken.class); + ContainerTokenIdentifier tokenidentifier = + new ContainerTokenIdentifier(container.getId(), + container.getContainerManagerAddress(), container.getResource()); + containerToken.setIdentifier(ByteBuffer.wrap(tokenidentifier.getBytes())); + containerToken.setKind(ContainerTokenIdentifier.KIND.toString()); + containerToken.setPassword(ByteBuffer.wrap(containerTokenSecretManager .createPassword(tokenidentifier))); containerToken.setService(container.getContainerManagerAddress()); container.setContainerToken(containerToken); @@ -1019,4 +1019,4 @@ public class LeafQueue implements Queue parent.recoverContainer(clusterResource, application, container); } -} \ No newline at end of file +} Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java?rev=1099337&r1=1099336&r2=1099337&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java Wed May 4 06:53:52 2011 @@ -51,10 +51,10 @@ import org.apache.hadoop.yarn.factory.pr import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ASMEvent; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationTrackerEventType; -import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.ApplicationInfo; import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.RMState; +import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.ClusterTracker; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Application; @@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; +import org.apache.hadoop.yarn.util.BuilderUtils; @LimitedPrivate("yarn") @Evolving @@ -86,9 +87,9 @@ public class FifoScheduler implements Re public static final Resource MINIMUM_ALLOCATION = Resources.createResource(MINIMUM_MEMORY); - Map<ApplicationId, Application> applications = - new TreeMap<ApplicationId, Application>( - new org.apache.hadoop.yarn.util.BuilderUtils.ApplicationIdComparator()); + Map<ApplicationId, Application> applications = + new TreeMap<ApplicationId, Application>( + new BuilderUtils.ApplicationIdComparator()); private static final String DEFAULT_QUEUE_NAME = "default"; private final QueueMetrics metrics = @@ -437,10 +438,10 @@ public class FifoScheduler implements Re new ArrayList<Container>(assignedContainers); for (int i=0; i < assignedContainers; ++i) { Container container = - org.apache.hadoop.yarn.server.resourcemanager.resource.Container - .create(recordFactory, application.getApplicationId(), - application.getNewContainerId(), node.getNodeAddress(), - node.getHttpAddress(), capability); + BuilderUtils.newContainer(recordFactory, + application.getApplicationId(), + application.getNewContainerId(), node.getNodeAddress(), + node.getHttpAddress(), capability); // If security is enabled, send the container-tokens too. if (UserGroupInformation.isSecurityEnabled()) { ContainerToken containerToken = recordFactory.newRecordInstance(ContainerToken.class); Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java?rev=1099337&r1=1099336&r2=1099337&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/Application.java Wed May 4 06:53:52 2011 @@ -81,7 +81,7 @@ public class Application { new org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceRequest.Comparator()); final private Set<Container> release = new TreeSet<Container>( - new org.apache.hadoop.yarn.server.resourcemanager.resource.Container.ContainerComparator()); + new org.apache.hadoop.yarn.util.BuilderUtils.ContainerComparator()); final private Map<String, NodeManager> nodes = new HashMap<String, NodeManager>(); @@ -274,7 +274,7 @@ public class Application { for (Container container : response) { if (container.getState() != ContainerState.COMPLETE) { containers.add( - org.apache.hadoop.yarn.server.resourcemanager.resource.Container.create( + org.apache.hadoop.yarn.util.BuilderUtils.clone( container)); } } Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1099337&r1=1099336&r2=1099337&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java Wed May 4 06:53:52 2011 @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.api import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.RMResourceTrackerImpl; +import org.apache.hadoop.yarn.util.BuilderUtils; @Private public class NodeManager implements ContainerManager { @@ -165,11 +166,11 @@ public class NodeManager implements Cont } } - Container container = - org.apache.hadoop.yarn.server.resourcemanager.resource.Container - .create(containerLaunchContext.getContainerId(), - containerManagerAddress, nodeHttpAddress, - containerLaunchContext.getResource()); + Container container = + BuilderUtils.newContainer(containerLaunchContext.getContainerId(), + containerManagerAddress, nodeHttpAddress, + containerLaunchContext.getResource()); + applicationContainers.add(container); Resources.subtractFrom(available, containerLaunchContext.getResource()); Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java?rev=1099337&r1=1099336&r2=1099337&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestApplicationCleanup.java Wed May 4 06:53:52 2011 @@ -25,8 +25,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicInteger; - -import junit.framework.Assert; import junit.framework.TestCase; import org.apache.commons.logging.Log; @@ -41,16 +39,12 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.Container; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; -import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.YarnClusterMetrics; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.ApplicationTokenSecretManager; -import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest; -import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; -import org.apache.hadoop.yarn.server.api.records.NodeStatus; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ASMEvent; @@ -58,29 +52,26 @@ import org.apache.hadoop.yarn.server.res import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationEventType; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.ApplicationTrackerEventType; import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.events.ApplicationMasterEvents.SNEventType; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.ApplicationsStore.ApplicationStore; import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemStore; -import org.apache.hadoop.yarn.server.resourcemanager.recovery.Store.RMState; -import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.ClusterTracker; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfo; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.NodeInfoTracker; import org.apache.hadoop.yarn.server.resourcemanager.resourcetracker.RMResourceTrackerImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManager; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeManagerImpl; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeResponse; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceListener; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; /** * Testing application cleanup (notifications to nodemanagers). * */ +@Ignore public class TestApplicationCleanup extends TestCase { private static final Log LOG = LogFactory.getLog(TestApplicationCleanup.class); private static RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java?rev=1099337&r1=1099336&r2=1099337&view=diff ============================================================================== --- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java (original) +++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java Wed May 4 06:53:52 2011 @@ -149,9 +149,14 @@ public class MiniYARNCluster extends Com File logDir = new File(testWorkDir, MiniYARNCluster.this.getName() + "-logDir"); + File remoteLogDir = + new File(testWorkDir, MiniYARNCluster.this.getName() + + "-remoteLogDir"); logDir.mkdir(); + remoteLogDir.mkdir(); LOG.info("Created logDir in " + logDir.getAbsolutePath()); getConfig().set(NMConfig.NM_LOG_DIR, logDir.getAbsolutePath()); + getConfig().set(NMConfig.REMOTE_USER_LOG_DIR, remoteLogDir.getAbsolutePath()); nodeManager = new NodeManager() { @Override
