Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Tue Aug 19 23:49:39 2014 @@ -71,6 +71,17 @@ </property> <property> + <description> + The actual address the server will bind to. If this optional address is + set, the RPC and webapp servers will bind to this address and the port specified in + yarn.resourcemanager.address and yarn.resourcemanager.webapp.address, respectively. This + is most useful for making RM listen to all interfaces by setting to 0.0.0.0. + </description> + <name>yarn.resourcemanager.bind-host</name> + <value></value> + </property> + + <property> <description>The number of threads used to handle applications manager requests.</description> <name>yarn.resourcemanager.client.thread-count</name> <value>50</value> @@ -195,6 +206,15 @@ </property> <property> + <description>Flag to enable override of the default kerberos authentication + filter with the RM authentication filter to allow authentication using + delegation tokens(fallback to kerberos if the tokens are missing). Only + applicable when the http authentication type is kerberos.</description> + <name>yarn.resourcemanager.webapp.delegation-token-auth-filter.enabled</name> + <value>true</value> + </property> + + <property> <description>How long to wait until a node manager is considered dead.</description> <name>yarn.nm.liveness-monitor.expiry-interval-ms</name> <value>600000</value> @@ -270,6 +290,14 @@ </property> <property> + <description>Enable RM work preserving recovery. This configuration is private + to YARN for experimenting the feature. + </description> + <name>yarn.resourcemanager.work-preserving-recovery.enabled</name> + <value>false</value> + </property> + + <property> <description>The class to use as the persistent store. If org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore @@ -309,14 +337,17 @@ <property> <description>Number of times RM tries to connect to ZooKeeper.</description> <name>yarn.resourcemanager.zk-num-retries</name> - <value>500</value> + <value>1000</value> </property> <property> <description>Retry interval in milliseconds when connecting to ZooKeeper. + When HA is enabled, the value here is NOT used. It is generated + automatically from yarn.resourcemanager.zk-timeout-ms and + yarn.resourcemanager.zk-num-retries. </description> <name>yarn.resourcemanager.zk-retry-interval-ms</name> - <value>2000</value> + <value>1000</value> </property> <property> @@ -370,6 +401,18 @@ </property> <property> + <description> + Specify the auths to be used for the ACL's specified in both the + yarn.resourcemanager.zk-acl and + yarn.resourcemanager.zk-state-store.root-node.acl properties. This + takes a comma-separated list of authentication mechanisms, each of the + form 'scheme:auth' (the same syntax used for the 'addAuth' command in + the ZK CLI). + </description> + <name>yarn.resourcemanager.zk-auth</name> + </property> + + <property> <description>URI pointing to the location of the FileSystem path where RM state will be stored. This must be supplied when using org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore @@ -431,7 +474,7 @@ <property> <description>Name of the cluster. In a HA setting, this is used to ensure the RM participates in leader - election fo this cluster and ensures it does not affect + election for this cluster and ensures it does not affect other clusters</description> <name>yarn.resourcemanager.cluster-id</name> <!--value>yarn-cluster</value--> @@ -604,6 +647,17 @@ </property> <property> + <description> + The actual address the server will bind to. If this optional address is + set, the RPC and webapp servers will bind to this address and the port specified in + yarn.nodemanager.address and yarn.nodemanager.webapp.address, respectively. This is + most useful for making NM listen to all interfaces by setting to 0.0.0.0. + </description> + <name>yarn.nodemanager.bind-host</name> + <value></value> + </property> + + <property> <description>Environment variables that should be forwarded from the NodeManager's environment to the container's.</description> <name>yarn.nodemanager.admin-env</name> <value>MALLOC_ARENA_MAX=$MALLOC_ARENA_MAX</value> @@ -733,7 +787,14 @@ </property> <property> - <description>Whether to enable log aggregation</description> + <description>Whether to enable log aggregation. Log aggregation collects + each container's logs and moves these logs onto a file-system, for e.g. + HDFS, after the application completes. Users can configure the + "yarn.nodemanager.remote-app-log-dir" and + "yarn.nodemanager.remote-app-log-dir-suffix" properties to determine + where these logs are moved to. Users can access the logs via the + Application Timeline Server. + </description> <name>yarn.log-aggregation-enable</name> <value>false</value> </property> @@ -1105,7 +1166,7 @@ If enabled, clients will put entities and events to the timeline server. </description> <name>yarn.timeline-service.enabled</name> - <value>true</value> + <value>false</value> </property> <property> @@ -1134,9 +1195,21 @@ </property> <property> + <description> + The actual address the server will bind to. If this optional address is + set, the RPC and webapp servers will bind to this address and the port specified in + yarn.timeline-service.address and yarn.timeline-service.webapp.address, respectively. + This is most useful for making the service listen to all interfaces by setting to + 0.0.0.0. + </description> + <name>yarn.timeline-service.bind-host</name> + <value></value> + </property> + + <property> <description>Store class name for timeline store.</description> <name>yarn.timeline-service.store-class</name> - <value>org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore</value> + <value>org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore</value> </property> <property> @@ -1154,7 +1227,7 @@ <property> <description>Store file name for leveldb timeline store.</description> <name>yarn.timeline-service.leveldb-timeline-store.path</name> - <value>${yarn.log.dir}/timeline</value> + <value>${hadoop.tmp.dir}/yarn/timeline</value> </property> <property> @@ -1188,6 +1261,36 @@ </property> <property> + <name>yarn.timeline-service.http-authentication.type</name> + <value>simple</value> + <description> + Defines authentication used for the timeline server HTTP endpoint. + Supported values are: simple | kerberos | #AUTHENTICATION_HANDLER_CLASSNAME# + </description> + </property> + + <property> + <name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name> + <value>true</value> + <description> + Indicates if anonymous requests are allowed by the timeline server when using + 'simple' authentication. + </description> + </property> + + <property> + <description>The Kerberos principal for the timeline server.</description> + <name>yarn.timeline-service.principal</name> + <value></value> + </property> + + <property> + <description>The Kerberos keytab for the timeline server.</description> + <name>yarn.timeline-service.keytab</name> + <value>/etc/krb5.keytab</value> + </property> + + <property> <description>Indicate to ResourceManager as well as clients whether history-service is enabled or not. If enabled, ResourceManager starts recording historical data that ApplicationHistory service can consume. @@ -1203,7 +1306,7 @@ org.apache.hadoop.yarn.server.applicationhistoryservice.FileSystemApplicationHistoryStore as the value for yarn.timeline-service.generic-application-history.store-class</description> <name>yarn.timeline-service.generic-application-history.fs-history-store.uri</name> - <value>${hadoop.log.dir}/yarn/system/history</value> + <value>${hadoop.tmp.dir}/yarn/timeline/generic-history</value> </property> <property>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java Tue Aug 19 23:49:39 2014 @@ -24,8 +24,6 @@ import java.net.SocketTimeoutException; import java.util.ArrayList; import java.util.List; -import org.junit.Assert; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -46,6 +44,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -55,6 +54,7 @@ import org.apache.hadoop.yarn.factory.pr import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC; import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; +import org.junit.Assert; import org.junit.Test; /* @@ -102,7 +102,8 @@ public class TestContainerLaunchRPC { Resource resource = Resource.newInstance(1234, 2); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", - resource, System.currentTimeMillis() + 10000, 42, 42); + resource, System.currentTimeMillis() + 10000, 42, 42, + Priority.newInstance(0), 0); Token containerToken = TestRPC.newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java Tue Aug 19 23:49:39 2014 @@ -23,8 +23,6 @@ import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.List; -import org.junit.Assert; - import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.io.Text; import org.apache.hadoop.ipc.ProtobufRpcEngine; @@ -50,6 +48,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerState; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.Token; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -61,6 +60,7 @@ import org.apache.hadoop.yarn.ipc.RPCUti import org.apache.hadoop.yarn.ipc.YarnRPC; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.util.Records; +import org.junit.Assert; import org.junit.Test; public class TestRPC { @@ -129,7 +129,8 @@ public class TestRPC { Resource resource = Resource.newInstance(1234, 2); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", - resource, System.currentTimeMillis() + 10000, 42, 42); + resource, System.currentTimeMillis() + 10000, 42, 42, + Priority.newInstance(0), 0); Token containerToken = newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfiguration.java Tue Aug 19 23:49:39 2014 @@ -28,6 +28,7 @@ import java.net.SocketAddress; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertFalse; public class TestYarnConfiguration { @@ -75,4 +76,131 @@ public class TestYarnConfiguration { YarnConfiguration.DEFAULT_NM_PORT); assertEquals(1234, addr.getPort()); } + + @Test + public void testGetSocketAddr() throws Exception { + + YarnConfiguration conf; + InetSocketAddress resourceTrackerAddress; + + //all default + conf = new YarnConfiguration(); + resourceTrackerAddress = conf.getSocketAddr( + YarnConfiguration.RM_BIND_HOST, + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); + assertEquals( + new InetSocketAddress( + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0], + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), + resourceTrackerAddress); + + //with address + conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.1"); + resourceTrackerAddress = conf.getSocketAddr( + YarnConfiguration.RM_BIND_HOST, + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); + assertEquals( + new InetSocketAddress( + "10.0.0.1", + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), + resourceTrackerAddress); + + //address and socket + conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.2:5001"); + resourceTrackerAddress = conf.getSocketAddr( + YarnConfiguration.RM_BIND_HOST, + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); + assertEquals( + new InetSocketAddress( + "10.0.0.2", + 5001), + resourceTrackerAddress); + + //bind host only + conf = new YarnConfiguration(); + conf.set(YarnConfiguration.RM_BIND_HOST, "10.0.0.3"); + resourceTrackerAddress = conf.getSocketAddr( + YarnConfiguration.RM_BIND_HOST, + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); + assertEquals( + new InetSocketAddress( + "10.0.0.3", + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), + resourceTrackerAddress); + + //bind host and address no port + conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0"); + conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.2"); + resourceTrackerAddress = conf.getSocketAddr( + YarnConfiguration.RM_BIND_HOST, + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); + assertEquals( + new InetSocketAddress( + "0.0.0.0", + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT), + resourceTrackerAddress); + + //bind host and address with port + conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0"); + conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "10.0.0.2:5003"); + resourceTrackerAddress = conf.getSocketAddr( + YarnConfiguration.RM_BIND_HOST, + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_PORT); + assertEquals( + new InetSocketAddress( + "0.0.0.0", + 5003), + resourceTrackerAddress); + + } + + @Test + public void testUpdateConnectAddr() throws Exception { + YarnConfiguration conf; + InetSocketAddress resourceTrackerConnectAddress; + InetSocketAddress serverAddress; + + //no override, old behavior. Won't work on a host named "yo.yo.yo" + conf = new YarnConfiguration(); + conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo"); + serverAddress = new InetSocketAddress( + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0], + Integer.valueOf(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1])); + + resourceTrackerConnectAddress = conf.updateConnectAddr( + YarnConfiguration.RM_BIND_HOST, + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, + serverAddress); + + assertFalse(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo")); + + //cause override with address + conf = new YarnConfiguration(); + conf.set(YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, "yo.yo.yo"); + conf.set(YarnConfiguration.RM_BIND_HOST, "0.0.0.0"); + serverAddress = new InetSocketAddress( + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[0], + Integer.valueOf(YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS.split(":")[1])); + + resourceTrackerConnectAddress = conf.updateConnectAddr( + YarnConfiguration.RM_BIND_HOST, + YarnConfiguration.RM_RESOURCE_TRACKER_ADDRESS, + YarnConfiguration.DEFAULT_RM_RESOURCE_TRACKER_ADDRESS, + serverAddress); + + assertTrue(resourceTrackerConnectAddress.toString().startsWith("yo.yo.yo")); + } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java Tue Aug 19 23:49:39 2014 @@ -28,6 +28,7 @@ public class DrainDispatcher extends Asy // and similar grotesqueries private volatile boolean drained = false; private final BlockingQueue<Event> queue; + final Object mutex; public DrainDispatcher() { this(new LinkedBlockingQueue<Event>()); @@ -36,6 +37,7 @@ public class DrainDispatcher extends Asy private DrainDispatcher(BlockingQueue<Event> eventQueue) { super(eventQueue); this.queue = eventQueue; + this.mutex = this; } /** @@ -53,8 +55,10 @@ public class DrainDispatcher extends Asy @Override public void run() { while (!Thread.currentThread().isInterrupted()) { - // !drained if dispatch queued new events on this dispatcher - drained = queue.isEmpty(); + synchronized (mutex) { + // !drained if dispatch queued new events on this dispatcher + drained = queue.isEmpty(); + } Event event; try { event = queue.take(); @@ -75,8 +79,10 @@ public class DrainDispatcher extends Asy return new EventHandler() { @Override public void handle(Event event) { - drained = false; - actual.handle(event); + synchronized (mutex) { + actual.handle(event); + drained = false; + } } }; } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java Tue Aug 19 23:49:39 2014 @@ -23,6 +23,7 @@ import static org.apache.hadoop.fs.Creat import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; import java.io.File; import java.io.FileOutputStream; @@ -66,6 +67,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.LocalDirAllocator; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.records.LocalResource; import org.apache.hadoop.yarn.api.records.LocalResourceType; @@ -308,6 +310,11 @@ public class TestFSDownload { FileContext files = FileContext.getLocalFSFileContext(conf); Path basedir = files.makeQualified(new Path("target", TestFSDownload.class.getSimpleName())); + + // if test directory doesn't have ancestor permission, skip this test + FileSystem f = basedir.getFileSystem(conf); + assumeTrue(FSDownload.ancestorsHaveExecutePermissions(f, basedir, null)); + files.mkdir(basedir, null, true); conf.setStrings(TestFSDownload.class.getName(), basedir.toString()); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimes.java Tue Aug 19 23:49:39 2014 @@ -50,4 +50,15 @@ public class TestTimes { elapsed = Times.elapsed(5, 10, false); Assert.assertEquals("Elapsed time is not 5", 5, elapsed); } + + @Test + public void testFinishTimesAheadOfStartTimes() { + long elapsed = Times.elapsed(10, 5, true); + Assert.assertEquals("Elapsed time is not -1", -1, elapsed); + elapsed = Times.elapsed(10, 5, false); + Assert.assertEquals("Elapsed time is not -1", -1, elapsed); + // use Long.MAX_VALUE to ensure started time is after the current one + elapsed = Times.elapsed(Long.MAX_VALUE, 0, true); + Assert.assertEquals("Elapsed time is not -1", -1, elapsed); + } } \ No newline at end of file Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestSubViews.java Tue Aug 19 23:49:39 2014 @@ -70,6 +70,6 @@ public class TestSubViews { out.flush(); verify(out).print("sub1 text"); verify(out).print("sub2 text"); - verify(out, times(15)).println(); // test inline transition across views + verify(out, times(16)).println(); // test inline transition across views } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestHtmlPage.java Tue Aug 19 23:49:39 2014 @@ -22,11 +22,12 @@ import com.google.inject.Injector; import java.io.PrintWriter; +import org.apache.hadoop.yarn.webapp.MimeType; import org.apache.hadoop.yarn.webapp.WebAppException; import org.apache.hadoop.yarn.webapp.test.WebAppTests; import org.apache.hadoop.yarn.webapp.view.HtmlPage; - import org.junit.Test; + import static org.mockito.Mockito.*; public class TestHtmlPage { @@ -53,6 +54,12 @@ public class TestHtmlPage { Injector injector = WebAppTests.testPage(TestView.class); PrintWriter out = injector.getInstance(PrintWriter.class); + // Verify the HTML page has correct meta tags in the header + verify(out).print(" http-equiv=\"X-UA-Compatible\""); + verify(out).print(" content=\"IE=8\""); + verify(out).print(" http-equiv=\"Content-type\""); + verify(out).print(String.format(" content=\"%s\"", MimeType.HTML)); + verify(out).print("test"); verify(out).print(" id=\"testid\""); verify(out).print("test note"); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/view/TestInfoBlock.java Tue Aug 19 23:49:39 2014 @@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.webapp.vi import java.io.PrintWriter; import java.io.StringWriter; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import org.apache.hadoop.yarn.webapp.ResponseInfo; @@ -34,6 +35,33 @@ public class TestInfoBlock { public static PrintWriter pw; + static final String JAVASCRIPT = "<script>alert('text')</script>"; + static final String JAVASCRIPT_ESCAPED = + "<script>alert('text')</script>"; + + public static class JavaScriptInfoBlock extends InfoBlock{ + + static ResponseInfo resInfo; + + static { + resInfo = new ResponseInfo(); + resInfo._("User_Name", JAVASCRIPT); + } + + @Override + public PrintWriter writer() { + return TestInfoBlock.pw; + } + + JavaScriptInfoBlock(ResponseInfo info) { + super(resInfo); + } + + public JavaScriptInfoBlock() { + super(resInfo); + } + } + public static class MultilineInfoBlock extends InfoBlock{ static ResponseInfo resInfo; @@ -78,4 +106,13 @@ public class TestInfoBlock { + " This is second line.%n </div>%n"); assertTrue(output.contains(expectedSinglelineData) && output.contains(expectedMultilineData)); } + + @Test(timeout=60000L) + public void testJavaScriptInfoBlock() throws Exception{ + WebAppTests.testBlock(JavaScriptInfoBlock.class); + TestInfoBlock.pw.flush(); + String output = TestInfoBlock.sw.toString(); + assertFalse(output.contains("<script>")); + assertTrue(output.contains(JAVASCRIPT_ESCAPED)); + } } \ No newline at end of file Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml Tue Aug 19 23:49:39 2014 @@ -45,24 +45,6 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>commons-el</groupId> - <artifactId>commons-el</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-compiler</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-2.1-jetty</artifactId> - </exclusion> - </exclusions> </dependency> <!-- 'mvn dependency:analyze' fails to detect use of this dependency --> @@ -169,15 +151,6 @@ </dependency> <dependency> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-core-asl</artifactId> - </dependency> - <dependency> - <groupId>org.codehaus.jackson</groupId> - <artifactId>jackson-mapper-asl</artifactId> - </dependency> - - <dependency> <groupId>commons-collections</groupId> <artifactId>commons-collections</artifactId> </dependency> Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryClientService.java Tue Aug 19 23:49:39 2014 @@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.exceptions import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException; import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.ipc.RPCUtil; import org.apache.hadoop.yarn.ipc.YarnRPC; public class ApplicationHistoryClientService extends AbstractService { @@ -75,10 +76,11 @@ public class ApplicationHistoryClientSer protected void serviceStart() throws Exception { Configuration conf = getConfig(); YarnRPC rpc = YarnRPC.create(conf); - InetSocketAddress address = - conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, - YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); + InetSocketAddress address = conf.getSocketAddr( + YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, + YarnConfiguration.TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); server = rpc.getServer(ApplicationHistoryProtocol.class, protocolHandler, @@ -88,8 +90,10 @@ public class ApplicationHistoryClientSer server.start(); this.bindAddress = - conf.updateConnectAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, - server.getListenerAddress()); + conf.updateConnectAddr(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, + YarnConfiguration.TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, + server.getListenerAddress()); LOG.info("Instantiated ApplicationHistoryClientService at " + this.bindAddress); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java Tue Aug 19 23:49:39 2014 @@ -18,12 +18,18 @@ package org.apache.hadoop.yarn.server.applicationhistoryservice; +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; + import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; import org.apache.hadoop.metrics2.source.JvmMetrics; +import org.apache.hadoop.security.AuthenticationFilterInitializer; +import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.service.CompositeService; import org.apache.hadoop.service.Service; import org.apache.hadoop.util.ExitUtil; @@ -33,9 +39,13 @@ import org.apache.hadoop.util.StringUtil import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.LeveldbTimelineStore; import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; +import org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore; +import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; +import org.apache.hadoop.yarn.server.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager; +import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; +import org.apache.hadoop.yarn.server.timeline.security.TimelineDelegationTokenSecretManagerService; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.WebApps; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; @@ -52,9 +62,11 @@ public class ApplicationHistoryServer ex private static final Log LOG = LogFactory .getLog(ApplicationHistoryServer.class); - ApplicationHistoryClientService ahsClientService; - ApplicationHistoryManager historyManager; - TimelineStore timelineStore; + private ApplicationHistoryClientService ahsClientService; + private ApplicationHistoryManager historyManager; + private TimelineStore timelineStore; + private TimelineDelegationTokenSecretManagerService secretManagerService; + private TimelineDataManager timelineDataManager; private WebApp webApp; public ApplicationHistoryServer() { @@ -63,19 +75,31 @@ public class ApplicationHistoryServer ex @Override protected void serviceInit(Configuration conf) throws Exception { - historyManager = createApplicationHistory(); + // init timeline services first + timelineStore = createTimelineStore(conf); + addIfService(timelineStore); + secretManagerService = createTimelineDelegationTokenSecretManagerService(conf); + addService(secretManagerService); + timelineDataManager = createTimelineDataManager(conf); + + // init generic history service afterwards + historyManager = createApplicationHistoryManager(conf); ahsClientService = createApplicationHistoryClientService(historyManager); addService(ahsClientService); addService((Service) historyManager); - timelineStore = createTimelineStore(conf); - addIfService(timelineStore); + + DefaultMetricsSystem.initialize("ApplicationHistoryServer"); + JvmMetrics.initSingleton("ApplicationHistoryServer", null); super.serviceInit(conf); } @Override protected void serviceStart() throws Exception { - DefaultMetricsSystem.initialize("ApplicationHistoryServer"); - JvmMetrics.initSingleton("ApplicationHistoryServer", null); + try { + doSecureLogin(getConfig()); + } catch(IOException ie) { + throw new YarnRuntimeException("Failed to login", ie); + } startWebApp(); super.serviceStart(); @@ -93,21 +117,22 @@ public class ApplicationHistoryServer ex @Private @VisibleForTesting - public ApplicationHistoryClientService getClientService() { + ApplicationHistoryClientService getClientService() { return this.ahsClientService; } - protected ApplicationHistoryClientService - createApplicationHistoryClientService( - ApplicationHistoryManager historyManager) { - return new ApplicationHistoryClientService(historyManager); - } - - protected ApplicationHistoryManager createApplicationHistory() { - return new ApplicationHistoryManagerImpl(); + /** + * @return ApplicationTimelineStore + */ + @Private + @VisibleForTesting + public TimelineStore getTimelineStore() { + return timelineStore; } - protected ApplicationHistoryManager getApplicationHistory() { + @Private + @VisibleForTesting + ApplicationHistoryManager getApplicationHistoryManager() { return this.historyManager; } @@ -136,45 +161,104 @@ public class ApplicationHistoryServer ex launchAppHistoryServer(args); } - protected ApplicationHistoryManager createApplicationHistoryManager( + private ApplicationHistoryClientService + createApplicationHistoryClientService( + ApplicationHistoryManager historyManager) { + return new ApplicationHistoryClientService(historyManager); + } + + private ApplicationHistoryManager createApplicationHistoryManager( Configuration conf) { return new ApplicationHistoryManagerImpl(); } - protected TimelineStore createTimelineStore( + private TimelineStore createTimelineStore( Configuration conf) { return ReflectionUtils.newInstance(conf.getClass( YarnConfiguration.TIMELINE_SERVICE_STORE, LeveldbTimelineStore.class, TimelineStore.class), conf); } - protected void startWebApp() { - String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(getConfig()); + private TimelineDelegationTokenSecretManagerService + createTimelineDelegationTokenSecretManagerService(Configuration conf) { + return new TimelineDelegationTokenSecretManagerService(); + } + + private TimelineDataManager createTimelineDataManager(Configuration conf) { + return new TimelineDataManager( + timelineStore, new TimelineACLsManager(conf)); + } + + private void startWebApp() { + Configuration conf = getConfig(); + // Always load pseudo authentication filter to parse "user.name" in an URL + // to identify a HTTP request's user in insecure mode. + // When Kerberos authentication type is set (i.e., secure mode is turned on), + // the customized filter will be loaded by the timeline server to do Kerberos + // + DT authentication. + String initializers = conf.get("hadoop.http.filter.initializers"); + + initializers = + initializers == null || initializers.length() == 0 ? "" : initializers; + + if (!initializers.contains(TimelineAuthenticationFilterInitializer.class + .getName())) { + initializers = + TimelineAuthenticationFilterInitializer.class.getName() + "," + + initializers; + } + + String[] parts = initializers.split(","); + ArrayList<String> target = new ArrayList<String>(); + for (String filterInitializer : parts) { + filterInitializer = filterInitializer.trim(); + if (filterInitializer.equals(AuthenticationFilterInitializer.class + .getName())) { + continue; + } + target.add(filterInitializer); + } + String actualInitializers = + org.apache.commons.lang.StringUtils.join(target, ","); + if (!actualInitializers.equals(initializers)) { + conf.set("hadoop.http.filter.initializers", actualInitializers); + } + String bindAddress = WebAppUtils.getWebAppBindURL(conf, + YarnConfiguration.TIMELINE_SERVICE_BIND_HOST, + WebAppUtils.getAHSWebAppURLWithoutScheme(conf)); LOG.info("Instantiating AHSWebApp at " + bindAddress); try { + AHSWebApp ahsWebApp = AHSWebApp.getInstance(); + ahsWebApp.setApplicationHistoryManager(historyManager); + ahsWebApp.setTimelineDelegationTokenSecretManagerService(secretManagerService); + ahsWebApp.setTimelineDataManager(timelineDataManager); webApp = WebApps .$for("applicationhistory", ApplicationHistoryClientService.class, - ahsClientService, "ws") - .with(getConfig()) - .withHttpSpnegoPrincipalKey( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_USER_NAME_KEY) - .withHttpSpnegoKeytabKey( - YarnConfiguration.TIMELINE_SERVICE_WEBAPP_SPNEGO_KEYTAB_FILE_KEY) - .at(bindAddress) - .start(new AHSWebApp(historyManager, timelineStore)); + ahsClientService, "ws") + .with(conf).at(bindAddress).start(ahsWebApp); } catch (Exception e) { String msg = "AHSWebApp failed to start."; LOG.error(msg, e); throw new YarnRuntimeException(msg, e); } } + + private void doSecureLogin(Configuration conf) throws IOException { + InetSocketAddress socAddr = getBindAddress(conf); + SecurityUtil.login(conf, YarnConfiguration.TIMELINE_SERVICE_KEYTAB, + YarnConfiguration.TIMELINE_SERVICE_PRINCIPAL, socAddr.getHostName()); + } + /** - * @return ApplicationTimelineStore + * Retrieve the timeline server bind address from configuration + * + * @param conf + * @return InetSocketAddress */ - @Private - @VisibleForTesting - public TimelineStore getTimelineStore() { - return timelineStore; + private static InetSocketAddress getBindAddress(Configuration conf) { + return conf.getSocketAddr(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ADDRESS, + YarnConfiguration.DEFAULT_TIMELINE_SERVICE_PORT); } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/webapp/AHSWebApp.java Tue Aug 19 23:49:39 2014 @@ -19,23 +19,69 @@ package org.apache.hadoop.yarn.server.ap import static org.apache.hadoop.yarn.util.StringHelper.pajoin; +import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.yarn.server.api.ApplicationContext; import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryManager; -import org.apache.hadoop.yarn.server.applicationhistoryservice.timeline.TimelineStore; +import org.apache.hadoop.yarn.server.timeline.TimelineDataManager; +import org.apache.hadoop.yarn.server.timeline.security.TimelineDelegationTokenSecretManagerService; +import org.apache.hadoop.yarn.server.timeline.webapp.TimelineWebServices; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.WebApp; import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider; import org.apache.hadoop.yarn.webapp.YarnWebParams; +import com.google.common.annotations.VisibleForTesting; + public class AHSWebApp extends WebApp implements YarnWebParams { - private final ApplicationHistoryManager applicationHistoryManager; - private final TimelineStore timelineStore; + private ApplicationHistoryManager applicationHistoryManager; + private TimelineDelegationTokenSecretManagerService secretManagerService; + private TimelineDataManager timelineDataManager; + + private static AHSWebApp instance = null; + + public static AHSWebApp getInstance() { + if (instance == null) { + instance = new AHSWebApp(); + } + return instance; + } + + @Private + @VisibleForTesting + public static void resetInstance() { + instance = null; + } + + private AHSWebApp() { + + } + + public ApplicationHistoryManager getApplicationHistoryManager() { + return applicationHistoryManager; + } - public AHSWebApp(ApplicationHistoryManager applicationHistoryManager, - TimelineStore timelineStore) { + public void setApplicationHistoryManager( + ApplicationHistoryManager applicationHistoryManager) { this.applicationHistoryManager = applicationHistoryManager; - this.timelineStore = timelineStore; + } + + public TimelineDelegationTokenSecretManagerService + getTimelineDelegationTokenSecretManagerService() { + return secretManagerService; + } + + public void setTimelineDelegationTokenSecretManagerService( + TimelineDelegationTokenSecretManagerService secretManagerService) { + this.secretManagerService = secretManagerService; + } + + public TimelineDataManager getTimelineDataManager() { + return timelineDataManager; + } + + public void setTimelineDataManager(TimelineDataManager timelineDataManager) { + this.timelineDataManager = timelineDataManager; } @Override @@ -45,7 +91,9 @@ public class AHSWebApp extends WebApp im bind(TimelineWebServices.class); bind(GenericExceptionHandler.class); bind(ApplicationContext.class).toInstance(applicationHistoryManager); - bind(TimelineStore.class).toInstance(timelineStore); + bind(TimelineDelegationTokenSecretManagerService.class).toInstance( + secretManagerService); + bind(TimelineDataManager.class).toInstance(timelineDataManager); route("/", AHSController.class); route(pajoin("/apps", APP_STATE), AHSController.class); route(pajoin("/app", APPLICATION_ID), AHSController.class, "app"); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java Tue Aug 19 23:49:39 2014 @@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.record import org.apache.hadoop.yarn.api.records.ContainerReport; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.exceptions.YarnException; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.junit.After; import org.junit.Before; @@ -68,12 +69,13 @@ public class TestApplicationHistoryClien historyServer.init(config); historyServer.start(); store = - ((ApplicationHistoryManagerImpl) historyServer.getApplicationHistory()) + ((ApplicationHistoryManagerImpl) historyServer.getApplicationHistoryManager()) .getHistoryStore(); } @After public void tearDown() throws Exception { + AHSWebApp.resetInstance(); historyServer.stop(); } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java Tue Aug 19 23:49:39 2014 @@ -23,10 +23,14 @@ import static org.junit.Assert.assertNot import static org.junit.Assert.fail; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.security.AuthenticationFilterInitializer; import org.apache.hadoop.service.Service.STATE; import org.apache.hadoop.util.ExitUtil; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp; +import org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilterInitializer; import org.junit.After; +import org.junit.Assert; import org.junit.Test; public class TestApplicationHistoryServer { @@ -40,7 +44,7 @@ public class TestApplicationHistoryServe Configuration config = new YarnConfiguration(); historyServer.init(config); assertEquals(STATE.INITED, historyServer.getServiceState()); - assertEquals(3, historyServer.getServices().size()); + assertEquals(4, historyServer.getServices().size()); ApplicationHistoryClientService historyService = historyServer.getClientService(); assertNotNull(historyServer.getClientService()); @@ -68,10 +72,36 @@ public class TestApplicationHistoryServe } } + @Test(timeout = 50000) + public void testFilteOverrides() throws Exception { + + String[] filterInitializers = + { + AuthenticationFilterInitializer.class.getName(), + TimelineAuthenticationFilterInitializer.class.getName(), + AuthenticationFilterInitializer.class.getName() + "," + + TimelineAuthenticationFilterInitializer.class.getName(), + AuthenticationFilterInitializer.class.getName() + ", " + + TimelineAuthenticationFilterInitializer.class.getName() }; + for (String filterInitializer : filterInitializers) { + historyServer = new ApplicationHistoryServer(); + Configuration config = new YarnConfiguration(); + config.set("hadoop.http.filter.initializers", filterInitializer); + historyServer.init(config); + historyServer.start(); + Configuration tmp = historyServer.getConfig(); + assertEquals(TimelineAuthenticationFilterInitializer.class.getName(), + tmp.get("hadoop.http.filter.initializers")); + historyServer.stop(); + AHSWebApp.resetInstance(); + } + } + @After public void stop() { if (historyServer != null) { historyServer.stop(); } + AHSWebApp.resetInstance(); } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java Tue Aug 19 23:49:39 2014 @@ -198,7 +198,7 @@ public class TestMemoryApplicationHistor writeContainerFinishData(containerId); } long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb; - Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 200); + Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 400); } } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml Tue Aug 19 23:49:39 2014 @@ -38,24 +38,6 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>commons-el</groupId> - <artifactId>commons-el</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-compiler</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-2.1-jetty</artifactId> - </exclusion> - </exclusions> </dependency> <!-- 'mvn dependency:analyze' fails to detect use of this dependency --> <dependency> @@ -103,6 +85,10 @@ <groupId>org.apache.zookeeper</groupId> <artifactId>zookeeper</artifactId> </dependency> + <dependency> + <groupId>org.fusesource.leveldbjni</groupId> + <artifactId>leveldbjni-all</artifactId> + </dependency> </dependencies> <build> Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java Tue Aug 19 23:49:39 2014 @@ -20,16 +20,17 @@ package org.apache.hadoop.yarn.server.ap import java.util.List; -import org.apache.hadoop.yarn.api.records.ContainerStatus; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.util.Records; public abstract class RegisterNodeManagerRequest { - + public static RegisterNodeManagerRequest newInstance(NodeId nodeId, int httpPort, Resource resource, String nodeManagerVersionId, - List<ContainerStatus> containerStatuses) { + List<NMContainerStatus> containerStatuses, + List<ApplicationId> runningApplications) { RegisterNodeManagerRequest request = Records.newRecord(RegisterNodeManagerRequest.class); request.setHttpPort(httpPort); @@ -37,6 +38,7 @@ public abstract class RegisterNodeManage request.setNodeId(nodeId); request.setNMVersion(nodeManagerVersionId); request.setContainerStatuses(containerStatuses); + request.setRunningApplications(runningApplications); return request; } @@ -44,11 +46,32 @@ public abstract class RegisterNodeManage public abstract int getHttpPort(); public abstract Resource getResource(); public abstract String getNMVersion(); - public abstract List<ContainerStatus> getContainerStatuses(); + public abstract List<NMContainerStatus> getNMContainerStatuses(); + + /** + * We introduce this here because currently YARN RM doesn't persist nodes info + * for application running. When RM restart happened, we cannot determinate if + * a node should do application cleanup (like log-aggregation, status update, + * etc.) or not. <p/> + * When we have this running application list in node manager register + * request, we can recover nodes info for running applications. And then we + * can take actions accordingly + * + * @return running application list in this node + */ + public abstract List<ApplicationId> getRunningApplications(); public abstract void setNodeId(NodeId nodeId); public abstract void setHttpPort(int port); public abstract void setResource(Resource resource); public abstract void setNMVersion(String version); - public abstract void setContainerStatuses(List<ContainerStatus> containerStatuses); + public abstract void setContainerStatuses( + List<NMContainerStatus> containerStatuses); + + /** + * Setter for {@link RegisterNodeManagerRequest#getRunningApplications()} + * @param runningApplications running application in this node + */ + public abstract void setRunningApplications( + List<ApplicationId> runningApplications); } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java Tue Aug 19 23:49:39 2014 @@ -23,21 +23,26 @@ import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.NodeId; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; +import org.apache.hadoop.yarn.api.records.ApplicationId; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; -import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl; +import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl; import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl; -import org.apache.hadoop.yarn.api.records.impl.pb.ProtoBase; import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl; -import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto; +import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto; import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; +import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NMContainerStatusProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerRequestProto; import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.RegisterNodeManagerRequestProtoOrBuilder; +import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; @@ -49,7 +54,8 @@ public class RegisterNodeManagerRequestP private Resource resource = null; private NodeId nodeId = null; - private List<ContainerStatus> containerStatuses = null; + private List<NMContainerStatus> containerStatuses = null; + private List<ApplicationId> runningApplications = null; public RegisterNodeManagerRequestPBImpl() { builder = RegisterNodeManagerRequestProto.newBuilder(); @@ -69,7 +75,10 @@ public class RegisterNodeManagerRequestP private void mergeLocalToBuilder() { if (this.containerStatuses != null) { - addContainerStatusesToProto(); + addNMContainerStatusesToProto(); + } + if (this.runningApplications != null) { + addRunningApplicationsToProto(); } if (this.resource != null) { builder.setResource(convertToProtoFormat(this.resource)); @@ -80,6 +89,18 @@ public class RegisterNodeManagerRequestP } + private synchronized void addNMContainerStatusesToProto() { + maybeInitBuilder(); + builder.clearContainerStatuses(); + List<NMContainerStatusProto> list = + new ArrayList<NMContainerStatusProto>(); + for (NMContainerStatus status : this.containerStatuses) { + list.add(convertToProtoFormat(status)); + } + builder.addAllContainerStatuses(list); + } + + private void mergeLocalToProto() { if (viaProto) maybeInitBuilder(); @@ -152,46 +173,46 @@ public class RegisterNodeManagerRequestP maybeInitBuilder(); builder.setHttpPort(httpPort); } - + @Override - public List<ContainerStatus> getContainerStatuses() { - initContainerStatuses(); - return containerStatuses; + public List<ApplicationId> getRunningApplications() { + initRunningApplications(); + return runningApplications; } - private void initContainerStatuses() { - if (this.containerStatuses != null) { + private void initRunningApplications() { + if (this.runningApplications != null) { return; } RegisterNodeManagerRequestProtoOrBuilder p = viaProto ? proto : builder; - List<ContainerStatusProto> list = p.getContainerStatusesList(); - this.containerStatuses = new ArrayList<ContainerStatus>(); - for (ContainerStatusProto c : list) { - this.containerStatuses.add(convertFromProtoFormat(c)); + List<ApplicationIdProto> list = p.getRunningApplicationsList(); + this.runningApplications = new ArrayList<ApplicationId>(); + for (ApplicationIdProto c : list) { + this.runningApplications.add(convertFromProtoFormat(c)); } } @Override - public void setContainerStatuses(List<ContainerStatus> containers) { - if (containers == null) { + public void setRunningApplications(List<ApplicationId> apps) { + if (apps == null) { return; } - initContainerStatuses(); - this.containerStatuses.addAll(containers); + initRunningApplications(); + this.runningApplications.addAll(apps); } - private void addContainerStatusesToProto() { + private void addRunningApplicationsToProto() { maybeInitBuilder(); - builder.clearContainerStatuses(); - if (containerStatuses == null) { + builder.clearRunningApplications(); + if (runningApplications == null) { return; } - Iterable<ContainerStatusProto> it = new Iterable<ContainerStatusProto>() { + Iterable<ApplicationIdProto> it = new Iterable<ApplicationIdProto>() { @Override - public Iterator<ContainerStatusProto> iterator() { - return new Iterator<ContainerStatusProto>() { - Iterator<ContainerStatus> iter = containerStatuses.iterator(); + public Iterator<ApplicationIdProto> iterator() { + return new Iterator<ApplicationIdProto>() { + Iterator<ApplicationId> iter = runningApplications.iterator(); @Override public boolean hasNext() { @@ -199,7 +220,7 @@ public class RegisterNodeManagerRequestP } @Override - public ContainerStatusProto next() { + public ApplicationIdProto next() { return convertToProtoFormat(iter.next()); } @@ -210,7 +231,35 @@ public class RegisterNodeManagerRequestP }; } }; - builder.addAllContainerStatuses(it); + builder.addAllRunningApplications(it); + } + + @Override + public List<NMContainerStatus> getNMContainerStatuses() { + initContainerRecoveryReports(); + return containerStatuses; + } + + private void initContainerRecoveryReports() { + if (this.containerStatuses != null) { + return; + } + RegisterNodeManagerRequestProtoOrBuilder p = viaProto ? proto : builder; + List<NMContainerStatusProto> list = p.getContainerStatusesList(); + this.containerStatuses = new ArrayList<NMContainerStatus>(); + for (NMContainerStatusProto c : list) { + this.containerStatuses.add(convertFromProtoFormat(c)); + } + } + + @Override + public void setContainerStatuses( + List<NMContainerStatus> containerReports) { + if (containerReports == null) { + return; + } + initContainerRecoveryReports(); + this.containerStatuses.addAll(containerReports); } @Override @@ -242,6 +291,14 @@ public class RegisterNodeManagerRequestP maybeInitBuilder(); builder.setNmVersion(version); } + + private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) { + return new ApplicationIdPBImpl(p); + } + + private ApplicationIdProto convertToProtoFormat(ApplicationId t) { + return ((ApplicationIdPBImpl)t).getProto(); + } private NodeIdPBImpl convertFromProtoFormat(NodeIdProto p) { return new NodeIdPBImpl(p); @@ -259,11 +316,11 @@ public class RegisterNodeManagerRequestP return ((ResourcePBImpl)t).getProto(); } - private ContainerStatusPBImpl convertFromProtoFormat(ContainerStatusProto c) { - return new ContainerStatusPBImpl(c); + private NMContainerStatusPBImpl convertFromProtoFormat(NMContainerStatusProto c) { + return new NMContainerStatusPBImpl(c); } - private ContainerStatusProto convertToProtoFormat(ContainerStatus c) { - return ((ContainerStatusPBImpl)c).getProto(); + private NMContainerStatusProto convertToProtoFormat(NMContainerStatus c) { + return ((NMContainerStatusPBImpl)c).getProto(); } } \ No newline at end of file Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java Tue Aug 19 23:49:39 2014 @@ -43,7 +43,7 @@ public class BaseContainerTokenSecretMan private static Log LOG = LogFactory .getLog(BaseContainerTokenSecretManager.class); - private int serialNo = new SecureRandom().nextInt(); + protected int serialNo = new SecureRandom().nextInt(); protected final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); protected final Lock readLock = readWriteLock.readLock(); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java Tue Aug 19 23:49:39 2014 @@ -42,7 +42,7 @@ public class BaseNMTokenSecretManager ex private static Log LOG = LogFactory .getLog(BaseNMTokenSecretManager.class); - private int serialNo = new SecureRandom().nextInt(); + protected int serialNo = new SecureRandom().nextInt(); protected final ReadWriteLock readWriteLock = new ReentrantReadWriteLock(); protected final Lock readLock = readWriteLock.readLock(); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java Tue Aug 19 23:49:39 2014 @@ -156,8 +156,7 @@ public class BuilderUtils { byte[] password, long rmIdentifier) throws IOException { ContainerTokenIdentifier identifier = new ContainerTokenIdentifier(cId, host + ":" + port, user, r, - expiryTime, - masterKeyId, rmIdentifier); + expiryTime, masterKeyId, rmIdentifier, Priority.newInstance(0), 0); return newContainerToken(BuilderUtils.newNodeId(host, port), password, identifier); } Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java Tue Aug 19 23:49:39 2014 @@ -57,7 +57,7 @@ public class ContainerInfo { assignedNodeId = container.getAssignedNode().toString(); } priority = container.getPriority().getPriority(); - startedTime = container.getStartTime(); + startedTime = container.getCreationTime(); finishedTime = container.getFinishTime(); elapsedTime = Times.elapsed(startedTime, finishedTime); diagnosticsInfo = container.getDiagnosticsInfo(); Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto Tue Aug 19 23:49:39 2014 @@ -47,4 +47,10 @@ message NodeHealthStatusProto { optional bool is_node_healthy = 1; optional string health_report = 2; optional int64 last_health_report_time = 3; -} \ No newline at end of file +} + +message VersionProto { + optional int32 major_version = 1; + optional int32 minor_version = 2; +} + Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto Tue Aug 19 23:49:39 2014 @@ -30,7 +30,8 @@ message RegisterNodeManagerRequestProto optional int32 http_port = 3; optional ResourceProto resource = 4; optional string nm_version = 5; - repeated ContainerStatusProto containerStatuses = 6; + repeated NMContainerStatusProto container_statuses = 6; + repeated ApplicationIdProto runningApplications = 7; } message RegisterNodeManagerResponseProto { @@ -58,3 +59,13 @@ message NodeHeartbeatResponseProto { optional int64 nextHeartBeatInterval = 7; optional string diagnostics_message = 8; } + +message NMContainerStatusProto { + optional ContainerIdProto container_id = 1; + optional ContainerStateProto container_state = 2; + optional ResourceProto resource = 3; + optional PriorityProto priority = 4; + optional string diagnostics = 5 [default = "N/A"]; + optional int32 container_exit_status = 6; + optional int64 creation_time = 7; +} \ No newline at end of file Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml Tue Aug 19 23:49:39 2014 @@ -40,24 +40,6 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>commons-el</groupId> - <artifactId>commons-el</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-compiler</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-2.1-jetty</artifactId> - </exclusion> - </exclusions> </dependency> <!-- 'mvn dependency:analyze' fails to detect use of this dependency --> @@ -174,6 +156,10 @@ <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-yarn-server-common</artifactId> </dependency> + <dependency> + <groupId>org.fusesource.leveldbjni</groupId> + <artifactId>leveldbjni-all</artifactId> + </dependency> </dependencies> <profiles> @@ -310,6 +296,7 @@ <source> <directory>${basedir}/src/main/proto</directory> <includes> + <include>yarn_server_nodemanager_recovery.proto</include> <include>yarn_server_nodemanager_service_protos.proto</include> <include>LocalizationProtocol.proto</include> </includes> Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java?rev=1619012&r1=1619011&r2=1619012&view=diff ============================================================================== --- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java (original) +++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java Tue Aug 19 23:49:39 2014 @@ -18,6 +18,7 @@ package org.apache.hadoop.yarn.server.nodemanager; +import java.io.File; import java.io.IOException; import java.net.InetSocketAddress; import java.util.ArrayList; @@ -29,17 +30,18 @@ import java.util.concurrent.locks.Reentr import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; +import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader; import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; @@ -126,9 +128,76 @@ public abstract class ContainerExecutor public abstract void deleteAsUser(String user, Path subDir, Path... basedirs) throws IOException, InterruptedException; + public abstract boolean isContainerProcessAlive(String user, String pid) + throws IOException; + + /** + * Recover an already existing container. This is a blocking call and returns + * only when the container exits. Note that the container must have been + * activated prior to this call. + * @param user the user of the container + * @param containerId The ID of the container to reacquire + * @return The exit code of the pre-existing container + * @throws IOException + */ + public int reacquireContainer(String user, ContainerId containerId) + throws IOException { + Path pidPath = getPidFilePath(containerId); + if (pidPath == null) { + LOG.warn(containerId + " is not active, returning terminated error"); + return ExitCode.TERMINATED.getExitCode(); + } + + String pid = null; + pid = ProcessIdFileReader.getProcessId(pidPath); + if (pid == null) { + throw new IOException("Unable to determine pid for " + containerId); + } + + LOG.info("Reacquiring " + containerId + " with pid " + pid); + try { + while(isContainerProcessAlive(user, pid)) { + Thread.sleep(1000); + } + } catch (InterruptedException e) { + throw new IOException("Interrupted while waiting for process " + pid + + " to exit", e); + } + + // wait for exit code file to appear + String exitCodeFile = ContainerLaunch.getExitCodeFile(pidPath.toString()); + File file = new File(exitCodeFile); + final int sleepMsec = 100; + int msecLeft = 2000; + while (!file.exists() && msecLeft >= 0) { + if (!isContainerActive(containerId)) { + LOG.info(containerId + " was deactivated"); + return ExitCode.TERMINATED.getExitCode(); + } + try { + Thread.sleep(sleepMsec); + } catch (InterruptedException e) { + throw new IOException( + "Interrupted while waiting for exit code from " + containerId, e); + } + msecLeft -= sleepMsec; + } + if (msecLeft < 0) { + throw new IOException("Timeout while waiting for exit code from " + + containerId); + } + + try { + return Integer.parseInt(FileUtils.readFileToString(file).trim()); + } catch (NumberFormatException e) { + throw new IOException("Error parsing exit code from pid " + pid, e); + } + } + public enum ExitCode { FORCE_KILLED(137), - TERMINATED(143); + TERMINATED(143), + LOST(154); private final int code; private ExitCode(int exitCode) {
