Author: arp
Date: Thu Sep 26 21:49:18 2013
New Revision: 1526717
URL: http://svn.apache.org/r1526717
Log:
Merging r1526366 through r1526708 from trunk to branch HDFS-2832
Removed:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptImpl.java
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
Modified: hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/HDFS-2832/hadoop-yarn-project/CHANGES.txt Thu Sep 26
21:49:18 2013
@@ -36,6 +36,9 @@ Release 2.3.0 - UNRELEASED
YARN-353. Add Zookeeper-based store implementation for RMStateStore.
(Bikas Saha, Jian He and Karthik Kambatla via hitesh)
+ YARN-819. ResourceManager and NodeManager should check for a minimum
allowed
+ version (Robert Parker via jeagles)
+
OPTIMIZATIONS
BUG FIXES
@@ -83,6 +86,9 @@ Release 2.1.2 - UNRELEASED
YARN-49. Improve distributed shell application to work on a secure cluster.
(Vinod Kumar Vavilapalli via hitesh)
+ YARN-1157. Fixed ResourceManager UI to behave correctly when apps like
+ distributed-shell do not set tracking urls. (Xuan Gong via vinodkv)
+
Release 2.1.1-beta - 2013-09-23
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/FinishApplicationMasterRequest.java
Thu Sep 26 21:49:18 2013
@@ -100,11 +100,22 @@ public abstract class FinishApplicationM
public abstract String getTrackingUrl();
/**
- * Set the <em>tracking URL</em>for the <code>ApplicationMaster</code>
- * This url if contains scheme then that will be used by resource manager
- * web application proxy otherwise it will default to http.
- * @param url <em>tracking URL</em>for the
- * <code>ApplicationMaster</code>
+ * Set the <em>final tracking URL</em>for the <code>ApplicationMaster</code>.
+ * This is the web-URL to which ResourceManager or web-application proxy will
+ * redirect client/users once the application is finished and the
+ * <code>ApplicationMaster</code> is gone.
+ * <p>
+ * If the passed url has a scheme then that will be used by the
+ * ResourceManager and web-application proxy, otherwise the scheme will
+ * default to http.
+ * </p>
+ * <p>
+ * Empty, null, "N/A" strings are all valid besides a real URL. In case an
url
+ * isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+ * <p>
+ *
+ * @param url
+ * <em>tracking URL</em>for the <code>ApplicationMaster</code>
*/
@Public
@Stable
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterRequest.java
Thu Sep 26 21:49:18 2013
@@ -112,11 +112,22 @@ public abstract class RegisterApplicatio
public abstract String getTrackingUrl();
/**
- * Set the <em>tracking URL</em> for the <code>ApplicationMaster</code>.
- * This url if contains scheme then that will be used by resource manager
- * web application proxy otherwise it will default to http.
- * @param trackingUrl <em>tracking URL</em> for the
- * <code>ApplicationMaster</code>
+ * Set the <em>tracking URL</em>for the <code>ApplicationMaster</code> while
+ * it is running. This is the web-URL to which ResourceManager or
+ * web-application proxy will redirect client/users while the application and
+ * the <code>ApplicationMaster</code> are still running.
+ * <p>
+ * If the passed url has a scheme then that will be used by the
+ * ResourceManager and web-application proxy, otherwise the scheme will
+ * default to http.
+ * </p>
+ * <p>
+ * Empty, null, "N/A" strings are all valid besides a real URL. In case an
url
+ * isn't explicitly passed, it defaults to "N/A" on the ResourceManager.
+ * <p>
+ *
+ * @param trackingUrl
+ * <em>tracking URL</em>for the <code>ApplicationMaster</code>
*/
@Public
@Stable
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
Thu Sep 26 21:49:18 2013
@@ -362,6 +362,13 @@ public class YarnConfiguration extends C
public static final long DEFAULT_RM_NMTOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS
=
24 * 60 * 60;
+
+ public static final String RM_NODEMANAGER_MINIMUM_VERSION =
+ RM_PREFIX + "nodemanager.minimum.version";
+
+ public static final String DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION =
+ "NONE";
+
////////////////////////////////
// Node Manager Configs
////////////////////////////////
@@ -460,6 +467,10 @@ public class YarnConfiguration extends C
public static final String NM_LOG_DIRS = NM_PREFIX + "log-dirs";
public static final String DEFAULT_NM_LOG_DIRS = "/tmp/logs";
+ public static final String NM_RESOURCEMANAGER_MINIMUM_VERSION =
+ NM_PREFIX + "resourcemanager.minimum.version";
+ public static final String DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION =
"NONE";
+
/** Interval at which the delayed token removal thread runs */
public static final String RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS =
RM_PREFIX + "delayed.delegation-token.removal-interval-ms";
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientContainerRequest.java
Thu Sep 26 21:49:18 2013
@@ -215,6 +215,10 @@ public class TestAMRMClientContainerRequ
@Override
public void reloadCachedMappings() {}
+
+ @Override
+ public void reloadCachedMappings(List<String> names) {
+ }
}
private void verifyResourceRequest(
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
Thu Sep 26 21:49:18 2013
@@ -359,6 +359,14 @@
</property>
<property>
+ <description>The minimum allowed version of a connecting nodemanager. The
valid values are
+ NONE (no version checking), EqualToRM (the nodemanager's version is
equal to
+ or greater than the RM version), or a Version String.</description>
+ <name>yarn.resourcemanager.nodemanager.minimum.version</name>
+ <value>NONE</value>
+ </property>
+
+ <property>
<description>Enable a set of periodic monitors (specified in
yarn.resourcemanager.scheduler.monitor.policies) that affect the
scheduler.</description>
@@ -738,6 +746,14 @@
</property>
<property>
+ <description>The minimum allowed version of a resourcemanager that a
nodemanager will connect to.
+ The valid values are NONE (no version checking), EqualToNM (the
resourcemanager's version is
+ equal to or greater than the NM version), or a Version
String.</description>
+ <name>yarn.nodemanager.resourcemanager.minimum.version</name>
+ <value>NONE</value>
+ </property>
+
+ <property>
<description>Max number of threads in NMClientAsync to process container
management events</description>
<name>yarn.client.nodemanager-client-async.thread-pool-max-size</name>
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestRackResolver.java
Thu Sep 26 21:49:18 2013
@@ -67,6 +67,10 @@ public class TestRackResolver {
public void reloadCachedMappings() {
// nothing to do here, since RawScriptBasedMapping has no cache.
}
+
+ @Override
+ public void reloadCachedMappings(List<String> names) {
+ }
}
@Test
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerRequest.java
Thu Sep 26 21:49:18 2013
@@ -25,8 +25,10 @@ public interface RegisterNodeManagerRequ
NodeId getNodeId();
int getHttpPort();
Resource getResource();
+ String getNMVersion();
void setNodeId(NodeId nodeId);
void setHttpPort(int port);
void setResource(Resource resource);
+ void setNMVersion(String version);
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/RegisterNodeManagerResponse.java
Thu Sep 26 21:49:18 2013
@@ -42,4 +42,7 @@ public interface RegisterNodeManagerResp
void setDiagnosticsMessage(String diagnosticsMessage);
+ void setRMVersion(String version);
+
+ String getRMVersion();
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerRequestPBImpl.java
Thu Sep 26 21:49:18 2013
@@ -139,6 +139,21 @@ public class RegisterNodeManagerRequestP
builder.setHttpPort(httpPort);
}
+ @Override
+ public String getNMVersion() {
+ RegisterNodeManagerRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasNmVersion()) {
+ return "";
+ }
+ return (p.getNmVersion());
+ }
+
+ @Override
+ public void setNMVersion(String version) {
+ maybeInitBuilder();
+ builder.setNmVersion(version);
+ }
+
private NodeIdPBImpl convertFromProtoFormat(NodeIdProto p) {
return new NodeIdPBImpl(p);
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RegisterNodeManagerResponsePBImpl.java
Thu Sep 26 21:49:18 2013
@@ -151,6 +151,25 @@ public class RegisterNodeManagerResponse
}
@Override
+ public String getRMVersion() {
+ RegisterNodeManagerResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasRmVersion()) {
+ return null;
+ }
+ return p.getRmVersion();
+ }
+
+ @Override
+ public void setRMVersion(String rmVersion) {
+ maybeInitBuilder();
+ if (rmVersion == null) {
+ builder.clearRmIdentifier();
+ return;
+ }
+ builder.setRmVersion(rmVersion);
+ }
+
+ @Override
public NodeAction getNodeAction() {
RegisterNodeManagerResponseProtoOrBuilder p = viaProto ? proto : builder;
if(!p.hasNodeAction()) {
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
Thu Sep 26 21:49:18 2013
@@ -29,6 +29,7 @@ message RegisterNodeManagerRequestProto
optional NodeIdProto node_id = 1;
optional int32 http_port = 3;
optional ResourceProto resource = 4;
+ optional string nm_version = 5;
}
message RegisterNodeManagerResponseProto {
@@ -37,6 +38,7 @@ message RegisterNodeManagerResponseProto
optional NodeActionProto nodeAction = 3;
optional int64 rm_identifier = 4;
optional string diagnostics_message = 5;
+ optional string rm_version = 6;
}
message NodeHeartbeatRequestProto {
@@ -45,7 +47,6 @@ message NodeHeartbeatRequestProto {
optional MasterKeyProto last_known_nm_token_master_key = 3;
}
-
message NodeHeartbeatResponseProto {
optional int32 response_id = 1;
optional MasterKeyProto container_token_master_key = 2;
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
Thu Sep 26 21:49:18 2013
@@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.VersionUtil;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -63,6 +64,7 @@ import org.apache.hadoop.yarn.server.api
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import
org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
import com.google.common.annotations.VisibleForTesting;
@@ -84,6 +86,8 @@ public class NodeStatusUpdaterImpl exten
private ResourceTracker resourceTracker;
private Resource totalResource;
private int httpPort;
+ private String nodeManagerVersionId;
+ private String minimumResourceManagerVersion;
private volatile boolean isStopped;
private RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
private boolean tokenKeepAliveEnabled;
@@ -138,6 +142,10 @@ public class NodeStatusUpdaterImpl exten
this.tokenRemovalDelayMs =
conf.getInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS);
+
+ this.minimumResourceManagerVersion = conf.get(
+ YarnConfiguration.NM_RESOURCEMANAGER_MINIMUM_VERSION,
+ YarnConfiguration.DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION);
// Default duration to track stopped containers on nodemanager is 10Min.
// This should not be assigned very large value as it will remember all the
@@ -168,6 +176,7 @@ public class NodeStatusUpdaterImpl exten
// NodeManager is the last service to start, so NodeId is available.
this.nodeId = this.context.getNodeId();
this.httpPort = this.context.getHttpPort();
+ this.nodeManagerVersionId = YarnVersionInfo.getVersion();
try {
// Registration has to be in start so that ContainerManager can get the
// perNM tokens needed to authenticate ContainerTokens.
@@ -235,6 +244,7 @@ public class NodeStatusUpdaterImpl exten
request.setHttpPort(this.httpPort);
request.setResource(this.totalResource);
request.setNodeId(this.nodeId);
+ request.setNMVersion(this.nodeManagerVersionId);
RegisterNodeManagerResponse regNMResponse =
resourceTracker.registerNodeManager(request);
this.rmIdentifier = regNMResponse.getRMIdentifier();
@@ -248,6 +258,26 @@ public class NodeStatusUpdaterImpl exten
+ message);
}
+ // if ResourceManager version is too old then shutdown
+ if (!minimumResourceManagerVersion.equals("NONE")){
+ if (minimumResourceManagerVersion.equals("EqualToNM")){
+ minimumResourceManagerVersion = nodeManagerVersionId;
+ }
+ String rmVersion = regNMResponse.getRMVersion();
+ if (rmVersion == null) {
+ String message = "The Resource Manager's did not return a version. "
+ + "Valid version cannot be checked.";
+ throw new YarnRuntimeException("Shutting down the Node Manager. "
+ + message);
+ }
+ if (VersionUtil.compareVersions(rmVersion,minimumResourceManagerVersion)
< 0) {
+ String message = "The Resource Manager's version ("
+ + rmVersion +") is less than the minimum "
+ + "allowed version " + minimumResourceManagerVersion;
+ throw new YarnRuntimeException("Shutting down the Node Manager on RM "
+ + "version error, " + message);
+ }
+ }
MasterKey masterKey = regNMResponse.getContainerTokenMasterKey();
// do this now so that its set before we start heartbeating to RM
// It is expected that status updater is started by this point and
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
Thu Sep 26 21:49:18 2013
@@ -145,7 +145,7 @@ public class TestNodeStatusUpdater {
.byteValue() }));
return masterKey;
}
-
+
private class MyResourceTracker implements ResourceTracker {
private final Context context;
@@ -471,6 +471,7 @@ public class TestNodeStatusUpdater {
public NodeAction heartBeatNodeAction = NodeAction.NORMAL;
public NodeAction registerNodeAction = NodeAction.NORMAL;
public String shutDownMessage = "";
+ public String rmVersion = "3.0.1";
@Override
public RegisterNodeManagerResponse registerNodeManager(
@@ -483,6 +484,7 @@ public class TestNodeStatusUpdater {
response.setContainerTokenMasterKey(createMasterKey());
response.setNMTokenMasterKey(createMasterKey());
response.setDiagnosticsMessage(shutDownMessage);
+ response.setRMVersion(rmVersion);
return response;
}
@Override
@@ -1180,6 +1182,44 @@ public class TestNodeStatusUpdater {
" connectionWaitSecs and RetryIntervalSecs", heartBeatID == 2);
}
+ @Test
+ public void testRMVersionLessThanMinimum() throws InterruptedException {
+ final AtomicInteger numCleanups = new AtomicInteger(0);
+ YarnConfiguration conf = createNMConfig();
+ conf.set(YarnConfiguration.NM_RESOURCEMANAGER_MINIMUM_VERSION, "3.0.0");
+ nm = new NodeManager() {
+ @Override
+ protected NodeStatusUpdater createNodeStatusUpdater(Context context,
+ Dispatcher
dispatcher, NodeHealthCheckerService healthChecker) {
+ MyNodeStatusUpdater myNodeStatusUpdater = new MyNodeStatusUpdater(
+ context, dispatcher, healthChecker, metrics);
+ MyResourceTracker2 myResourceTracker2 = new MyResourceTracker2();
+ myResourceTracker2.heartBeatNodeAction = NodeAction.NORMAL;
+ myResourceTracker2.rmVersion = "3.0.0";
+ myNodeStatusUpdater.resourceTracker = myResourceTracker2;
+ return myNodeStatusUpdater;
+ }
+
+ @Override
+ protected void cleanupContainers(NodeManagerEventType eventType) {
+ super.cleanupContainers(NodeManagerEventType.SHUTDOWN);
+ numCleanups.incrementAndGet();
+ }
+ };
+
+ nm.init(conf);
+ nm.start();
+
+ // NM takes a while to reach the STARTED state.
+ int waitCount = 0;
+ while (nm.getServiceState() != STATE.STARTED && waitCount++ != 20) {
+ LOG.info("Waiting for NM to stop..");
+ Thread.sleep(1000);
+ }
+ Assert.assertTrue(nm.getServiceState() == STATE.STARTED);
+ nm.stop();
+ }
+
private class MyNMContext extends NMContext {
ConcurrentMap<ContainerId, Container> containers =
new ConcurrentSkipListMap<ContainerId, Container>();
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
Thu Sep 26 21:49:18 2013
@@ -28,6 +28,7 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.VersionUtil;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -55,6 +56,7 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.security.authorize.RMPolicyProvider;
import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
import org.apache.hadoop.yarn.util.RackResolver;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
public class ResourceTrackerService extends AbstractService implements
ResourceTracker {
@@ -73,6 +75,7 @@ public class ResourceTrackerService exte
private long nextHeartBeatInterval;
private Server server;
private InetSocketAddress resourceTrackerAddress;
+ private String minimumNodeManagerVersion;
private static final NodeHeartbeatResponse resync = recordFactory
.newRecordInstance(NodeHeartbeatResponse.class);
@@ -99,6 +102,7 @@ public class ResourceTrackerService exte
this.nmLivelinessMonitor = nmLivelinessMonitor;
this.containerTokenSecretManager = containerTokenSecretManager;
this.nmTokenSecretManager = nmTokenSecretManager;
+
}
@Override
@@ -124,7 +128,11 @@ public class ResourceTrackerService exte
minAllocVcores = conf.getInt(
YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
-
+
+ minimumNodeManagerVersion = conf.get(
+ YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,
+ YarnConfiguration.DEFAULT_RM_NODEMANAGER_MINIMUM_VERSION);
+
super.serviceInit(conf);
}
@@ -172,10 +180,30 @@ public class ResourceTrackerService exte
int cmPort = nodeId.getPort();
int httpPort = request.getHttpPort();
Resource capability = request.getResource();
+ String nodeManagerVersion = request.getNMVersion();
RegisterNodeManagerResponse response = recordFactory
.newRecordInstance(RegisterNodeManagerResponse.class);
+ if (!minimumNodeManagerVersion.equals("NONE")) {
+ if (minimumNodeManagerVersion.equals("EqualToRM")) {
+ minimumNodeManagerVersion = YarnVersionInfo.getVersion();
+ }
+
+ if ((nodeManagerVersion == null) ||
+
(VersionUtil.compareVersions(nodeManagerVersion,minimumNodeManagerVersion)) <
0) {
+ String message =
+ "Disallowed NodeManager Version " + nodeManagerVersion
+ + ", is less than the minimum version "
+ + minimumNodeManagerVersion + " sending SHUTDOWN signal to "
+ + "NodeManager.";
+ LOG.info(message);
+ response.setDiagnosticsMessage(message);
+ response.setNodeAction(NodeAction.SHUTDOWN);
+ return response;
+ }
+ }
+
// Check if this node is a 'valid' node
if (!this.nodesListManager.isValidNode(host)) {
String message =
@@ -230,6 +258,7 @@ public class ResourceTrackerService exte
LOG.info(message);
response.setNodeAction(NodeAction.NORMAL);
response.setRMIdentifier(ResourceManager.getClusterTimeStamp());
+ response.setRMVersion(YarnVersionInfo.getVersion());
return response;
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
Thu Sep 26 21:49:18 2013
@@ -994,7 +994,7 @@ public class RMAppAttemptImpl implements
}
}
- static final class AMRegisteredTransition extends BaseTransition {
+ private static final class AMRegisteredTransition extends BaseTransition {
@Override
public void transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
@@ -1003,7 +1003,8 @@ public class RMAppAttemptImpl implements
= (RMAppAttemptRegistrationEvent) event;
appAttempt.host = registrationEvent.getHost();
appAttempt.rpcPort = registrationEvent.getRpcport();
- appAttempt.origTrackingUrl = registrationEvent.getTrackingurl();
+ appAttempt.origTrackingUrl =
+ sanitizeTrackingUrl(registrationEvent.getTrackingurl());
appAttempt.proxiedTrackingUrl =
appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
@@ -1138,7 +1139,8 @@ public class RMAppAttemptImpl implements
RMAppAttemptUnregistrationEvent unregisterEvent
= (RMAppAttemptUnregistrationEvent) event;
appAttempt.diagnostics.append(unregisterEvent.getDiagnostics());
- appAttempt.origTrackingUrl = unregisterEvent.getTrackingUrl();
+ appAttempt.origTrackingUrl =
+ sanitizeTrackingUrl(unregisterEvent.getTrackingUrl());
appAttempt.proxiedTrackingUrl =
appAttempt.generateProxyUriWithoutScheme(appAttempt.origTrackingUrl);
appAttempt.finalStatus = unregisterEvent.getFinalApplicationStatus();
@@ -1292,4 +1294,8 @@ public class RMAppAttemptImpl implements
appAttempt.rmContext.getAMRMTokenSecretManager()
.applicationMasterFinished(appAttempt.getAppAttemptId());
}
+
+ private static String sanitizeTrackingUrl(String url) {
+ return (url == null || url.trim().isEmpty()) ? "N/A" : url;
+ }
}
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
Thu Sep 26 21:49:18 2013
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
import org.apache.hadoop.yarn.util.resource.Resources;
@Private
@@ -96,9 +97,9 @@ public class NodeManager implements Cont
RegisterNodeManagerRequest request = recordFactory
.newRecordInstance(RegisterNodeManagerRequest.class);
request.setHttpPort(httpPort);
- request.setNodeId(this.nodeId);
request.setResource(capability);
request.setNodeId(this.nodeId);
+ request.setNMVersion(YarnVersionInfo.getVersion());
resourceTrackerService.registerNodeManager(request);
this.schedulerNode = new FiCaSchedulerNode(rmContext.getRMNodes().get(
this.nodeId), false);
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
Thu Sep 26 21:49:18 2013
@@ -46,6 +46,7 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.YarnVersionInfo;
import org.junit.After;
import org.junit.Test;
@@ -249,6 +250,59 @@ public class TestResourceTrackerService
}
@Test
+ public void testNodeRegistrationSuccess() throws Exception {
+ writeToHostsFile("host2");
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
+ .getAbsolutePath());
+ rm = new MockRM(conf);
+ rm.start();
+
+ ResourceTrackerService resourceTrackerService =
rm.getResourceTrackerService();
+ RegisterNodeManagerRequest req = Records.newRecord(
+ RegisterNodeManagerRequest.class);
+ NodeId nodeId = NodeId.newInstance("host2", 1234);
+ Resource capability = BuilderUtils.newResource(1024, 1);
+ req.setResource(capability);
+ req.setNodeId(nodeId);
+ req.setHttpPort(1234);
+ req.setNMVersion(YarnVersionInfo.getVersion());
+ // trying to register a invalid node.
+ RegisterNodeManagerResponse response =
resourceTrackerService.registerNodeManager(req);
+ Assert.assertEquals(NodeAction.NORMAL,response.getNodeAction());
+ }
+
+ @Test
+ public void testNodeRegistrationVersionLessThanRM() throws Exception {
+ writeToHostsFile("host2");
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH, hostFile
+ .getAbsolutePath());
+ conf.set(YarnConfiguration.RM_NODEMANAGER_MINIMUM_VERSION,"EqualToRM" );
+ rm = new MockRM(conf);
+ rm.start();
+ String nmVersion = "1.9.9";
+
+ ResourceTrackerService resourceTrackerService =
rm.getResourceTrackerService();
+ RegisterNodeManagerRequest req = Records.newRecord(
+ RegisterNodeManagerRequest.class);
+ NodeId nodeId = NodeId.newInstance("host2", 1234);
+ Resource capability = BuilderUtils.newResource(1024, 1);
+ req.setResource(capability);
+ req.setNodeId(nodeId);
+ req.setHttpPort(1234);
+ req.setNMVersion(nmVersion);
+ // trying to register a invalid node.
+ RegisterNodeManagerResponse response =
resourceTrackerService.registerNodeManager(req);
+ Assert.assertEquals(NodeAction.SHUTDOWN,response.getNodeAction());
+ Assert.assertTrue("Diagnostic message did not contain: 'Disallowed
NodeManager " +
+ "Version "+ nmVersion + ", is less than the minimum version'",
+ response.getDiagnosticsMessage().contains("Disallowed NodeManager
Version " +
+ nmVersion + ", is less than the minimum version "));
+
+ }
+
+ @Test
public void testNodeRegistrationFailure() throws Exception {
writeToHostsFile("host1");
Configuration conf = new Configuration();
Modified:
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1526717&r1=1526716&r2=1526717&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
(original)
+++
hadoop/common/branches/HDFS-2832/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
Thu Sep 26 21:49:18 2013
@@ -30,14 +30,18 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -85,8 +89,10 @@ import org.apache.hadoop.yarn.server.res
import
org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import
org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -261,8 +267,22 @@ public class TestRMAppAttemptTransitions
private String getProxyUrl(RMAppAttempt appAttempt) {
- return pjoin(RM_WEBAPP_ADDR, "proxy",
- appAttempt.getAppAttemptId().getApplicationId(), "");
+ String url = null;
+ try {
+ URI trackingUri =
+ StringUtils.isEmpty(appAttempt.getOriginalTrackingUrl()) ? null :
+ ProxyUriUtils
+ .getUriFromAMUrl(appAttempt.getOriginalTrackingUrl());
+ String proxy = WebAppUtils.getProxyHostAndPort(conf);
+ URI proxyUri = ProxyUriUtils.getUriFromAMUrl(proxy);
+ URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
+ appAttempt.getAppAttemptId().getApplicationId());
+ url = result.toASCIIString().substring(
+ HttpConfig.getSchemePrefix().length());
+ } catch (URISyntaxException ex) {
+ Assert.fail();
+ }
+ return url;
}
/**
@@ -448,9 +468,9 @@ public class TestRMAppAttemptTransitions
assertEquals(container, applicationAttempt.getMasterContainer());
assertEquals(host, applicationAttempt.getHost());
assertEquals(rpcPort, applicationAttempt.getRpcPort());
- assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+ verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
if (unmanagedAM) {
- assertEquals("oldtrackingurl", applicationAttempt.getTrackingUrl());
+ verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
} else {
assertEquals(getProxyUrl(applicationAttempt),
applicationAttempt.getTrackingUrl());
@@ -468,7 +488,7 @@ public class TestRMAppAttemptTransitions
assertEquals(RMAppAttemptState.FINISHING,
applicationAttempt.getAppAttemptState());
assertEquals(diagnostics, applicationAttempt.getDiagnostics());
- assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+ verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
assertEquals(getProxyUrl(applicationAttempt),
applicationAttempt.getTrackingUrl());
assertEquals(container, applicationAttempt.getMasterContainer());
@@ -487,9 +507,9 @@ public class TestRMAppAttemptTransitions
assertEquals(RMAppAttemptState.FINISHED,
applicationAttempt.getAppAttemptState());
assertEquals(diagnostics, applicationAttempt.getDiagnostics());
- assertEquals(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
+ verifyUrl(trackingUrl, applicationAttempt.getOriginalTrackingUrl());
if (unmanagedAM) {
- assertEquals("mytrackingurl", applicationAttempt.getTrackingUrl());
+ verifyUrl(trackingUrl, applicationAttempt.getTrackingUrl());
} else {
assertEquals(getProxyUrl(applicationAttempt),
@@ -603,9 +623,7 @@ public class TestRMAppAttemptTransitions
trackingUrl, diagnostics);
}
-
- @Test
- public void testUnmanagedAMSuccess() {
+ private void testUnmanagedAMSuccess(String url) {
unmanagedAM = true;
when(submissionContext.getUnmanagedAM()).thenReturn(true);
// submit AM and check it goes to LAUNCHED state
@@ -615,7 +633,7 @@ public class TestRMAppAttemptTransitions
applicationAttempt.getAppAttemptId());
// launch AM
- runApplicationAttempt(null, "host", 8042, "oldtrackingurl", true);
+ runApplicationAttempt(null, "host", 8042, url, true);
// complete a container
applicationAttempt.handle(new RMAppAttemptContainerAcquiredEvent(
@@ -623,13 +641,12 @@ public class TestRMAppAttemptTransitions
applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
applicationAttempt.getAppAttemptId(), mock(ContainerStatus.class)));
// complete AM
- String trackingUrl = "mytrackingurl";
String diagnostics = "Successful";
FinalApplicationStatus finalStatus = FinalApplicationStatus.SUCCEEDED;
applicationAttempt.handle(new RMAppAttemptUnregistrationEvent(
- applicationAttempt.getAppAttemptId(), trackingUrl, finalStatus,
+ applicationAttempt.getAppAttemptId(), url, finalStatus,
diagnostics));
- testAppAttemptFinishedState(null, finalStatus, trackingUrl, diagnostics, 1,
+ testAppAttemptFinishedState(null, finalStatus, url, diagnostics, 1,
true);
}
@@ -824,12 +841,42 @@ public class TestRMAppAttemptTransitions
"Killed by user");
}
+ @Test
+ public void testTrackingUrlUnmanagedAM() {
+ testUnmanagedAMSuccess("oldTrackingUrl");
+ }
@Test
- public void testNoTrackingUrl() {
+ public void testEmptyTrackingUrlUnmanagedAM() {
+ testUnmanagedAMSuccess("");
+ }
+
+ @Test
+ public void testNullTrackingUrlUnmanagedAM() {
+ testUnmanagedAMSuccess(null);
+ }
+
+ @Test
+ public void testManagedAMWithTrackingUrl() {
+ testTrackingUrlManagedAM("theTrackingUrl");
+ }
+
+ @Test
+ public void testManagedAMWithEmptyTrackingUrl() {
+ testTrackingUrlManagedAM("");
+ }
+
+ @Test
+ public void testManagedAMWithNullTrackingUrl() {
+ testTrackingUrlManagedAM(null);
+ }
+
+ private void testTrackingUrlManagedAM(String url) {
Container amContainer = allocateApplicationAttempt();
launchApplicationAttempt(amContainer);
- runApplicationAttempt(amContainer, "host", 8042, "", false);
+ runApplicationAttempt(amContainer, "host", 8042, url, false);
+ unregisterApplicationAttempt(amContainer,
+ FinalApplicationStatus.SUCCEEDED, url, "Successful");
}
@Test
@@ -927,4 +974,12 @@ public class TestRMAppAttemptTransitions
}
}
}
+
+ private void verifyUrl(String url1, String url2) {
+ if (url1 == null || url1.trim().isEmpty()) {
+ assertEquals("N/A", url2);
+ } else {
+ assertEquals(url1, url2);
+ }
+ }
}
