Author: arp
Date: Sun Sep 22 18:23:22 2013
New Revision: 1525410
URL: http://svn.apache.org/r1525410
Log:
Merging r1524587 through r1525408 from trunk to branch HDFS-2832.
Added:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
- copied unchanged from r1525408,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
- copied unchanged from r1525408,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestZKRMStateStoreZKClientConnections.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestEmptyQueues.java
- copied unchanged from r1525408,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/TestEmptyQueues.java
Removed:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/federation-background.gif
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/federation.gif
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt Sun Sep 22
18:23:22 2013
@@ -27,10 +27,15 @@ Release 2.3.0 - UNRELEASED
IMPROVEMENTS
YARN-905. Add state filters to nodes CLI (Wei Yan via Sandy Ryza)
+
YARN-1098. Separate out RM services into Always On and Active (Karthik
Kambatla via bikas)
+
YARN-1027. Implement RMHAProtocolService (Karthik Kambatla via bikas)
+ YARN-353. Add Zookeeper-based store implementation for RMStateStore.
+ (Bikas Saha, Jian He and Karthik Kambatla via hitesh)
+
OPTIMIZATIONS
BUG FIXES
@@ -50,6 +55,9 @@ Release 2.2.0 - UNRELEASED
BUG FIXES
+ YARN-1128. FifoPolicy.computeShares throws NPE on empty list of
Schedulables
+ (Karthik Kambatla via Sandy Ryza)
+
Release 2.1.1-beta - 2013-09-23
INCOMPATIBLE CHANGES
@@ -121,6 +129,9 @@ Release 2.1.1-beta - 2013-09-23
YARN-1001. Added a web-service to get statistics about per application-type
per state for consumption by downstream projects. (Zhijie Shen via vinodkv)
+ YARN-1203. Changed YARN web-app proxy to handle http and https URLs from
+ AM registration and finish correctly. (Omkar Vinit Joshi via vinodkv)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
Sun Sep 22 18:23:22 2013
@@ -91,6 +91,8 @@ public abstract class FinishApplicationM
/**
* Get the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ * This url if contains scheme then that will be used by resource manager
+ * web application proxy otherwise it will default to http.
* @return <em>tracking URL</em>for the <code>ApplicationMaster</code>
*/
@Public
@@ -99,6 +101,8 @@ public abstract class FinishApplicationM
/**
* Set the <em>tracking URL</em>for the <code>ApplicationMaster</code>
+ * This url if contains scheme then that will be used by resource manager
+ * web application proxy otherwise it will default to http.
* @param url <em>tracking URL</em>for the
* <code>ApplicationMaster</code>
*/
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
Sun Sep 22 18:23:22 2013
@@ -103,6 +103,8 @@ public abstract class RegisterApplicatio
/**
* Get the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ * This url if contains scheme then that will be used by resource manager
+ * web application proxy otherwise it will default to http.
* @return <em>tracking URL</em> for the <code>ApplicationMaster</code>
*/
@Public
@@ -111,6 +113,8 @@ public abstract class RegisterApplicatio
/**
* Set the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
+ * This url if contains scheme then that will be used by resource manager
+ * web application proxy otherwise it will default to http.
* @param trackingUrl <em>tracking URL</em> for the
* <code>ApplicationMaster</code>
*/
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Sun Sep 22 18:23:22 2013
@@ -26,6 +26,8 @@ import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
@@ -276,12 +278,40 @@ public class YarnConfiguration extends C
public static final String RM_HA_ENABLED = RM_HA_PREFIX + "enabled";
public static final boolean DEFAULT_RM_HA_ENABLED = false;
+
+ ////////////////////////////////
+ // RM state store configs
+ ////////////////////////////////
/** The class to use as the persistent store.*/
public static final String RM_STORE = RM_PREFIX + "store.class";
/** URI for FileSystemRMStateStore */
public static final String FS_RM_STATE_STORE_URI =
RM_PREFIX + "fs.state-store.uri";
+ /**
+ * Comma separated host:port pairs, each corresponding to a ZK server for
+ * ZKRMStateStore
+ */
+ public static final String ZK_STATE_STORE_PREFIX =
+ RM_PREFIX + "zk.state-store.";
+ public static final String ZK_RM_STATE_STORE_NUM_RETRIES =
+ ZK_STATE_STORE_PREFIX + "num-retries";
+ public static final int DEFAULT_ZK_RM_STATE_STORE_NUM_RETRIES = 3;
+ public static final String ZK_RM_STATE_STORE_ADDRESS =
+ ZK_STATE_STORE_PREFIX + "address";
+ /** Timeout in millisec for ZK server connection for ZKRMStateStore */
+ public static final String ZK_RM_STATE_STORE_TIMEOUT_MS =
+ ZK_STATE_STORE_PREFIX + "timeout.ms";
+ public static final int DEFAULT_ZK_RM_STATE_STORE_TIMEOUT_MS = 60000;
+ /** Parent znode path under which ZKRMStateStore will create znodes */
+ public static final String ZK_RM_STATE_STORE_PARENT_PATH =
+ ZK_STATE_STORE_PREFIX + "parent-path";
+ public static final String DEFAULT_ZK_RM_STATE_STORE_PARENT_PATH =
"/rmstore";
+ /** ACL for znodes in ZKRMStateStore */
+ public static final String ZK_RM_STATE_STORE_ACL =
+ ZK_STATE_STORE_PREFIX + "acl";
+ public static final String DEFAULT_ZK_RM_STATE_STORE_ACL =
+ "world:anyone:rwcda";
/** The maximum number of completed applications RM keeps. */
public static final String RM_MAX_COMPLETED_APPLICATIONS =
@@ -837,7 +867,8 @@ public class YarnConfiguration extends C
}
public static String getRMWebAppURL(Configuration conf) {
- return JOINER.join("http://";, getRMWebAppHostAndPort(conf));
+ return JOINER.join(HttpConfig.getSchemePrefix(),
+ getRMWebAppHostAndPort(conf));
}
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
Sun Sep 22 18:23:22 2013
@@ -259,6 +259,51 @@
</property>
<property>
+ <description>Host:Port of the ZooKeeper server where RM state will
+ be stored. This must be supplied when using
+ org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
+ as the value for yarn.resourcemanager.store.class</description>
+ <name>yarn.resourcemanager.zk.state-store.address</name>
+ <!--value>127.0.0.1:2181</value-->
+ </property>
+
+ <property>
+ <description>Number of times ZKRMStateStore tries to connect to
+ ZooKeeper. This may be supplied when using
+ org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
+ as the value for yarn.resourcemanager.store.class</description>
+ <name>yarn.resourcemanager.zk.state-store.num-retries</name>
+ <value>3</value>
+ </property>
+
+ <property>
+ <description>Full path of the ZooKeeper znode where RM state will be
+ stored. This must be supplied when using
+ org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
+ as the value for yarn.resourcemanager.store.class</description>
+ <name>yarn.resourcemanager.zk.state-store.parent-path</name>
+ <value>/rmstore</value>
+ </property>
+
+ <property>
+ <description>Timeout when connecting to ZooKeeper.
+ This may be supplied when using
+ org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
+ as the value for yarn.resourcemanager.store.class</description>
+ <name>yarn.resourcemanager.zk.state-store.timeout.ms</name>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <description>ACL's to be used for ZooKeeper znodes.
+ This may be supplied when using
+ org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore
+ as the value for yarn.resourcemanager.store.class</description>
+ <name>yarn.resourcemanager.zk.state-store.acl</name>
+ <value>world:anyone:rwcda</value>
+ </property>
+
+ <property>
<description>URI pointing to the location of the FileSystem path where
RM state will be stored. This must be supplied when using
org.apache.hadoop.yarn.server.resourcemanager.recovery.FileSystemRMStateStore
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
Sun Sep 22 18:23:22 2013
@@ -42,6 +42,16 @@
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
</dependency>
<dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
<type>test-jar</type>
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/FileSystemRMStateStore.java
Sun Sep 22 18:23:22 2013
@@ -63,12 +63,6 @@ public class FileSystemRMStateStore exte
public static final Log LOG =
LogFactory.getLog(FileSystemRMStateStore.class);
private static final String ROOT_DIR_NAME = "FSRMStateRoot";
- private static final String RM_DT_SECRET_MANAGER_ROOT =
"RMDTSecretManagerRoot";
- private static final String RM_APP_ROOT = "RMAppRoot";
- private static final String DELEGATION_KEY_PREFIX = "DelegationKey_";
- private static final String DELEGATION_TOKEN_PREFIX = "RMDelegationToken_";
- private static final String DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX =
- "RMDTSequenceNumber_";
protected FileSystem fs;
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
Sun Sep 22 18:23:22 2013
@@ -65,6 +65,14 @@ import org.apache.hadoop.yarn.server.res
*/
public abstract class RMStateStore extends AbstractService {
+ // constants for RM App state and RMDTSecretManagerState.
+ protected static final String RM_APP_ROOT = "RMAppRoot";
+ protected static final String RM_DT_SECRET_MANAGER_ROOT =
"RMDTSecretManagerRoot";
+ protected static final String DELEGATION_KEY_PREFIX = "DelegationKey_";
+ protected static final String DELEGATION_TOKEN_PREFIX = "RMDelegationToken_";
+ protected static final String DELEGATION_TOKEN_SEQUENCE_NUMBER_PREFIX =
+ "RMDTSequenceNumber_";
+
public static final Log LOG = LogFactory.getLog(RMStateStore.class);
public RMStateStore() {
@@ -464,8 +472,9 @@ public abstract class RMStateStore exten
(ApplicationAttemptStateDataPBImpl)
ApplicationAttemptStateDataPBImpl
.newApplicationAttemptStateData(attemptState.getAttemptId(),
attemptState.getMasterContainer(), appAttemptTokens);
-
- LOG.info("Storing info for attempt: " +
attemptState.getAttemptId());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Storing info for attempt: " +
attemptState.getAttemptId());
+ }
storeApplicationAttemptState(attemptState.getAttemptId().toString(),
attemptStateData);
} catch (Exception e) {
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FifoPolicy.java
Sun Sep 22 18:23:22 2013
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.re
import java.io.Serializable;
import java.util.Collection;
import java.util.Comparator;
+import java.util.Iterator;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -73,6 +74,10 @@ public class FifoPolicy extends Scheduli
@Override
public void computeShares(Collection<? extends Schedulable> schedulables,
Resource totalResources) {
+ if (schedulables.isEmpty()) {
+ return;
+ }
+
Schedulable earliest = null;
for (Schedulable schedulable : schedulables) {
if (earliest == null ||
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/TestRMStateStore.java
Sun Sep 22 18:23:22 2013
@@ -26,8 +26,10 @@ import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import javax.crypto.SecretKey;
@@ -40,6 +42,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ha.ClientBaseWithFixes;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.Text;
@@ -67,13 +70,17 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import
org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.util.ConverterUtils;
+
+import org.apache.zookeeper.ZooKeeper;
+
import org.junit.Test;
-public class TestRMStateStore {
+public class TestRMStateStore extends ClientBaseWithFixes{
public static final Log LOG = LogFactory.getLog(TestRMStateStore.class);
- class TestDispatcher implements Dispatcher,
EventHandler<RMAppAttemptStoredEvent> {
+ static class TestDispatcher implements
+ Dispatcher, EventHandler<RMAppAttemptStoredEvent> {
ApplicationAttemptId attemptId;
Exception storedException;
@@ -82,7 +89,8 @@ public class TestRMStateStore {
@SuppressWarnings("rawtypes")
@Override
- public void register(Class<? extends Enum> eventType, EventHandler
handler) {
+ public void register(Class<? extends Enum> eventType,
+ EventHandler handler) {
}
@Override
@@ -109,9 +117,17 @@ public class TestRMStateStore {
}
@Test
+ public void testZKRMStateStoreRealZK() throws Exception {
+ TestZKRMStateStoreTester zkTester = new TestZKRMStateStoreTester();
+ testRMAppStateStore(zkTester);
+ testRMDTSecretManagerStateStore(zkTester);
+ }
+
+ @Test
public void testFSRMStateStore() throws Exception {
HdfsConfiguration conf = new HdfsConfiguration();
- MiniDFSCluster cluster = new
MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ MiniDFSCluster cluster =
+ new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
try {
TestFSRMStateStoreTester fsTester = new
TestFSRMStateStoreTester(cluster);
testRMAppStateStore(fsTester);
@@ -121,6 +137,41 @@ public class TestRMStateStore {
}
}
+ class TestZKRMStateStoreTester implements RMStateStoreHelper {
+ ZooKeeper client;
+ ZKRMStateStore store;
+
+ class TestZKRMStateStore extends ZKRMStateStore {
+ public TestZKRMStateStore(Configuration conf, String workingZnode)
+ throws Exception {
+ init(conf);
+ start();
+ assertTrue(znodeWorkingPath.equals(workingZnode));
+ }
+
+ @Override
+ public ZooKeeper getNewZooKeeper() throws IOException {
+ return client;
+ }
+ }
+
+ public RMStateStore getRMStateStore() throws Exception {
+ String workingZnode = "/Test";
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.set(YarnConfiguration.ZK_RM_STATE_STORE_ADDRESS, hostPort);
+ conf.set(YarnConfiguration.ZK_RM_STATE_STORE_PARENT_PATH, workingZnode);
+ this.client = createClient();
+ this.store = new TestZKRMStateStore(conf, workingZnode);
+ return this.store;
+ }
+
+ @Override
+ public boolean isFinalStateValid() throws Exception {
+ List<String> nodes = client.getChildren(store.znodeWorkingPath, false);
+ return nodes.size() == 1;
+ }
+ }
+
class TestFSRMStateStoreTester implements RMStateStoreHelper {
Path workingDirPathURI;
FileSystemRMStateStore store;
@@ -149,7 +200,8 @@ public class TestRMStateStore {
@Override
public RMStateStore getRMStateStore() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
- conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI,
workingDirPathURI.toString());
+ conf.set(YarnConfiguration.FS_RM_STATE_STORE_URI,
+ workingDirPathURI.toString());
this.store = new TestFileSystemRMStore(conf);
return store;
}
@@ -158,11 +210,7 @@ public class TestRMStateStore {
public boolean isFinalStateValid() throws Exception {
FileSystem fs = cluster.getFileSystem();
FileStatus[] files = fs.listStatus(workingDirPathURI);
- if(files.length == 1) {
- // only store root directory should exist
- return true;
- }
- return false;
+ return files.length == 1;
}
}
@@ -183,9 +231,10 @@ public class TestRMStateStore {
dispatcher.notified = false;
}
- void storeApp(RMStateStore store, ApplicationId appId, long time)
- throws Exception
{
- ApplicationSubmissionContext context = new
ApplicationSubmissionContextPBImpl();
+ void storeApp(
+ RMStateStore store, ApplicationId appId, long time) throws Exception {
+ ApplicationSubmissionContext context =
+ new ApplicationSubmissionContextPBImpl();
context.setApplicationId(appId);
RMApp mockApp = mock(RMApp.class);
@@ -216,7 +265,8 @@ public class TestRMStateStore {
return container.getId();
}
- void testRMAppStateStore(RMStateStoreHelper stateStoreHelper) throws
Exception {
+ void testRMAppStateStore(RMStateStoreHelper stateStoreHelper)
+ throws Exception {
long submitTime = System.currentTimeMillis();
Configuration conf = new YarnConfiguration();
RMStateStore store = stateStoreHelper.getRMStateStore();
@@ -271,7 +321,8 @@ public class TestRMStateStore {
RMApp mockRemovedApp = mock(RMApp.class);
HashMap<ApplicationAttemptId, RMAppAttempt> attempts =
new HashMap<ApplicationAttemptId,
RMAppAttempt>();
- ApplicationSubmissionContext context = new
ApplicationSubmissionContextPBImpl();
+ ApplicationSubmissionContext context =
+ new ApplicationSubmissionContextPBImpl();
context.setApplicationId(appIdRemoved);
when(mockRemovedApp.getSubmitTime()).thenReturn(submitTime);
when(mockRemovedApp.getApplicationSubmissionContext()).thenReturn(context);
@@ -288,7 +339,8 @@ public class TestRMStateStore {
// load state
store = stateStoreHelper.getRMStateStore();
RMState state = store.loadState();
- Map<ApplicationId, ApplicationState> rmAppState =
state.getApplicationState();
+ Map<ApplicationId, ApplicationState> rmAppState =
+ state.getApplicationState();
ApplicationState appState = rmAppState.get(appId1);
// app is loaded
@@ -362,7 +414,8 @@ public class TestRMStateStore {
store.loadState().getRMDTSecretManagerState();
Assert.assertEquals(token1, secretManagerState.getTokenState());
Assert.assertEquals(keySet, secretManagerState.getMasterKeyState());
- Assert.assertEquals(sequenceNumber,
secretManagerState.getDTSequenceNumber());
+ Assert.assertEquals(sequenceNumber,
+ secretManagerState.getDTSequenceNumber());
}
private Token<AMRMTokenIdentifier> generateAMRMToken(
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/ProxyUriUtils.java
Sun Sep 22 18:23:22 2013
@@ -137,14 +137,44 @@ public class ProxyUriUtils {
/**
* Create a URI form a no scheme Url, such as is returned by the AM.
- * @param noSchemeUrl the URL formate returned by an AM
+ * @param url the URL format returned by an AM. This may or may not contain
+ * scheme.
* @return a URI with an http scheme
* @throws URISyntaxException if the url is not formatted correctly.
*/
- public static URI getUriFromAMUrl(String noSchemeUrl)
+ public static URI getUriFromAMUrl(String url)
throws URISyntaxException {
- return new URI(HttpConfig.getSchemePrefix() + noSchemeUrl);
+ if (getSchemeFromUrl(url).isEmpty()) {
+ /*
+ * check is made to make sure if AM reports with scheme then it will be
+ * used by default otherwise it will default to the one configured using
+ * "hadoop.ssl.enabled".
+ */
+ return new URI(HttpConfig.getSchemePrefix() + url);
+ } else {
+ return new URI(url);
+ }
}
+
+ /**
+ * Create a URI form a no scheme Url, such as is returned by the AM.
+ * @param noSchemeUrl the URL formate returned by an AM
+ * @return a URI with an http scheme
+ * @throws URISyntaxException if the url is not formatted correctly.
+ */
+ public static URI getUriFromAMUrl(String scheme, String noSchemeUrl)
+ throws URISyntaxException {
+ if (getSchemeFromUrl(noSchemeUrl).isEmpty()) {
+ /*
+ * check is made to make sure if AM reports with scheme then it will be
+ * used by default otherwise it will default to the one configured
using
+ * "hadoop.ssl.enabled".
+ */
+ return new URI(scheme + "://" + noSchemeUrl);
+ } else {
+ return new URI(noSchemeUrl);
+ }
+ }
/**
* Returns the first valid tracking link, if any, from the given id from the
@@ -169,4 +199,20 @@ public class ProxyUriUtils {
}
return null;
}
+
+ /**
+ * Returns the scheme if present in the url
+ * eg. "https://issues.apache.org/jira/browse/YARN"; > "https"
+ */
+ public static String getSchemeFromUrl(String url) {
+ int index = 0;
+ if (url != null) {
+ index = url.indexOf("://");
+ }
+ if (index > 0) {
+ return url.substring(0, index);
+ } else {
+ return "";
+ }
+ }
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java?rev=1525410&r1=1525409&r2=1525410&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/src/main/java/org/apache/hadoop/yarn/server/webproxy/WebAppProxyServlet.java
Sun Sep 22 18:23:22 2013
@@ -163,7 +163,6 @@ public class WebAppProxyServlet extends
}
config.setLocalAddress(localAddress);
HttpMethod method = new GetMethod(uri.getEscapedURI());
-
@SuppressWarnings("unchecked")
Enumeration<String> names = req.getHeaderNames();
while(names.hasMoreElements()) {
@@ -293,14 +292,17 @@ public class WebAppProxyServlet extends
}
String original = applicationReport.getOriginalTrackingUrl();
URI trackingUri = null;
- if (original != null) {
- trackingUri = ProxyUriUtils.getUriFromAMUrl(original);
- }
// fallback to ResourceManager's app page if no tracking URI provided
if(original == null || original.equals("N/A")) {
resp.sendRedirect(resp.encodeRedirectURL(
StringHelper.pjoin(rmAppPageUrlBase, id.toString())));
return;
+ } else {
+ if (ProxyUriUtils.getSchemeFromUrl(original).isEmpty()) {
+ trackingUri = ProxyUriUtils.getUriFromAMUrl("http", original);
+ } else {
+ trackingUri = new URI(original);
+ }
}
String runningUser = applicationReport.getUser();
@@ -311,8 +313,7 @@ public class WebAppProxyServlet extends
req.getQueryString(), true), runningUser, id);
return;
}
-
- URI toFetch = new URI(req.getScheme(),
+ URI toFetch = new URI(trackingUri.getScheme(),
trackingUri.getAuthority(),
StringHelper.ujoin(trackingUri.getPath(), rest),
req.getQueryString(),
null);
