Author: abayer
Date: Fri Aug 17 17:38:31 2012
New Revision: 1374376
URL: http://svn.apache.org/viewvc?rev=1374376&view=rev
Log:
WHIRR-629. Fix YARN tests on Rackspace (contributed by Tom White)
Modified:
whirr/trunk/CHANGES.txt
whirr/trunk/recipes/hadoop-yarn-cdh-ec2.properties
whirr/trunk/services/cdh/src/test/resources/whirr-hadoop-mr2-cdh-test.properties
whirr/trunk/services/yarn/pom.xml
whirr/trunk/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java
whirr/trunk/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java
whirr/trunk/services/yarn/src/main/resources/functions/configure_hadoop_mr2.sh
whirr/trunk/services/yarn/src/main/resources/whirr-yarn-default.properties
Modified: whirr/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/whirr/trunk/CHANGES.txt?rev=1374376&r1=1374375&r2=1374376&view=diff
==============================================================================
--- whirr/trunk/CHANGES.txt (original)
+++ whirr/trunk/CHANGES.txt Fri Aug 17 17:38:31 2012
@@ -185,6 +185,8 @@ Trunk (unreleased changes)
WHIRR-613. OpenJDK JAVA_HOME detection needs to be improved (Andrew Bayer
via adriancole)
+ WHIRR-629. YARN tests fail on Rackspace. (tomwhite)
+
Release 0.7.1 - 2012-02-23
IMPROVEMENTS
Modified: whirr/trunk/recipes/hadoop-yarn-cdh-ec2.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/recipes/hadoop-yarn-cdh-ec2.properties?rev=1374376&r1=1374375&r2=1374376&view=diff
==============================================================================
--- whirr/trunk/recipes/hadoop-yarn-cdh-ec2.properties (original)
+++ whirr/trunk/recipes/hadoop-yarn-cdh-ec2.properties Fri Aug 17 17:38:31 2012
@@ -30,13 +30,14 @@ whirr.instance-templates=1 hadoop-nameno
# We need to use modified scripts for the installation since it has changed
# significantly since 0.20.x
-whirr.java.install-function=install_oracle_jdk6
+whirr.java.install-function=install_openjdk
whirr.hadoop.install-function=install_cdh_hadoop
whirr.hadoop.configure-function=configure_cdh_hadoop
whirr.yarn.configure-function=configure_cdh_yarn
whirr.yarn.start-function=start_cdh_yarn
whirr.mr_jobhistory.start-function=start_cdh_mr_jobhistory
whirr.env.REPO=cdh4
+whirr.env.MAPREDUCE_VERSION=2
# For EC2 set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment
variables.
whirr.provider=aws-ec2
@@ -69,3 +70,5 @@ whirr.credential=${env:AWS_SECRET_ACCESS
hadoop-mapreduce.mapreduce.framework.name=yarn
hadoop-common.ipc.client.connect.max.retries=100
+hadoop-yarn.yarn.application.classpath=$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$YARN_HOME/*,$YARN_HOME/lib/*
+
Modified:
whirr/trunk/services/cdh/src/test/resources/whirr-hadoop-mr2-cdh-test.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/cdh/src/test/resources/whirr-hadoop-mr2-cdh-test.properties?rev=1374376&r1=1374375&r2=1374376&view=diff
==============================================================================
---
whirr/trunk/services/cdh/src/test/resources/whirr-hadoop-mr2-cdh-test.properties
(original)
+++
whirr/trunk/services/cdh/src/test/resources/whirr-hadoop-mr2-cdh-test.properties
Fri Aug 17 17:38:31 2012
@@ -34,4 +34,4 @@ whirr.env.MAPREDUCE_VERSION=2
hadoop-mapreduce.mapreduce.framework.name=yarn
hadoop-common.ipc.client.connect.max.retries=100
-
+hadoop-yarn.yarn.application.classpath=$HADOOP_CONF_DIR,$HADOOP_COMMON_HOME/*,$HADOOP_COMMON_HOME/lib/*,$HADOOP_HDFS_HOME/*,$HADOOP_HDFS_HOME/lib/*,$HADOOP_MAPRED_HOME/*,$HADOOP_MAPRED_HOME/lib/*,$YARN_HOME/*,$YARN_HOME/lib/*
Modified: whirr/trunk/services/yarn/pom.xml
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/yarn/pom.xml?rev=1374376&r1=1374375&r2=1374376&view=diff
==============================================================================
--- whirr/trunk/services/yarn/pom.xml (original)
+++ whirr/trunk/services/yarn/pom.xml Fri Aug 17 17:38:31 2012
@@ -95,13 +95,25 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>0.23.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-common</artifactId>
+ <version>0.23.1</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
<version>0.23.1</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-hdfs</artifactId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<version>0.23.1</version>
<scope>test</scope>
</dependency>
Modified:
whirr/trunk/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java?rev=1374376&r1=1374375&r2=1374376&view=diff
==============================================================================
---
whirr/trunk/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java
(original)
+++
whirr/trunk/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnConfigurationBuilder.java
Fri Aug 17 17:38:31 2012
@@ -44,6 +44,7 @@ public class YarnConfigurationBuilder {
throws ConfigurationException {
CompositeConfiguration config = new CompositeConfiguration();
Configuration sub = clusterSpec.getConfigurationForKeysWithPrefix(prefix);
+ config.setDelimiterParsingDisabled(true);
config.addConfiguration(sub.subset(prefix)); // remove prefix
config.addConfiguration(defaults.subset(prefix));
return config;
@@ -63,16 +64,22 @@ public class YarnConfigurationBuilder {
Configuration config = build(clusterSpec, cluster, defaults,
"hadoop-yarn");
- Instance resourceManager = cluster
- .getInstanceMatching(role(YarnResourceManagerHandler.ROLE));
- String resourceManagerPrivateAddress =
- resourceManager.getPrivateAddress().getHostName();
- config.setProperty("yarn.resourcemanager.address",
- String.format("%s:8040", resourceManagerPrivateAddress));
- config.setProperty("yarn.resourcemanager.scheduler.address",
- String.format("%s:8030", resourceManagerPrivateAddress));
- config.setProperty("yarn.resourcemanager.resource-tracker.address",
- String.format("%s:8025", resourceManagerPrivateAddress));
+ if (role.equals(YarnResourceManagerHandler.ROLE)) {
+ config.setProperty("yarn.resourcemanager.address", "0.0.0.0:8040");
+ config.setProperty("yarn.resourcemanager.scheduler.address",
"0.0.0.0:8030");
+ config.setProperty("yarn.resourcemanager.resource-tracker.address",
"0.0.0.0:8025");
+ } else {
+ Instance resourceManager = cluster
+ .getInstanceMatching(role(YarnResourceManagerHandler.ROLE));
+ String resourceManagerPrivateAddress =
+ resourceManager.getPrivateAddress().getHostName();
+ config.setProperty("yarn.resourcemanager.address",
+ String.format("%s:8040", resourceManagerPrivateAddress));
+ config.setProperty("yarn.resourcemanager.scheduler.address",
+ String.format("%s:8030", resourceManagerPrivateAddress));
+ config.setProperty("yarn.resourcemanager.resource-tracker.address",
+ String.format("%s:8025", resourceManagerPrivateAddress));
+ }
return config;
}
}
Modified:
whirr/trunk/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java?rev=1374376&r1=1374375&r2=1374376&view=diff
==============================================================================
---
whirr/trunk/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java
(original)
+++
whirr/trunk/services/yarn/src/main/java/org/apache/whirr/service/yarn/YarnResourceManagerHandler.java
Fri Aug 17 17:38:31 2012
@@ -24,10 +24,13 @@ import static org.jclouds.scriptbuilder.
import java.io.File;
import java.io.IOException;
+import java.util.Iterator;
import java.util.Properties;
+import org.apache.commons.configuration.AbstractConfiguration;
import org.apache.commons.configuration.Configuration;
import org.apache.commons.configuration.ConfigurationException;
+import org.apache.commons.lang.StringUtils;
import org.apache.whirr.Cluster;
import org.apache.whirr.Cluster.Instance;
import org.apache.whirr.ClusterSpec;
@@ -112,6 +115,18 @@ public class YarnResourceManagerHandler
private Properties createClientSideYarnProperties(ClusterSpec clusterSpec,
Instance resourceManager) throws IOException {
Properties config = new Properties();
+ String prefix = "hadoop-yarn";
+ Configuration sub = clusterSpec.getConfigurationForKeysWithPrefix(prefix);
+ sub = sub.subset(prefix); // remove prefix
+ for (@SuppressWarnings("unchecked")
+ Iterator<String> it = sub.getKeys(); it.hasNext(); ) {
+ String key = it.next();
+ // rebuild the original value by joining all of them with the default
separator
+ String value = StringUtils.join(sub.getStringArray(key),
+ AbstractConfiguration.getDefaultListDelimiter());
+ config.setProperty(key, value);
+ System.out.println("createClientSideYarnProperties " + key + ":" +
value);
+ }
config.setProperty("yarn.resourcemanager.address",
String.format("%s:8040", resourceManager.getPublicHostName()));
config.setProperty("yarn.resourcemanager.scheduler.address",
@@ -119,17 +134,6 @@ public class YarnResourceManagerHandler
config.setProperty("yarn.resourcemanager.resource-tracker.address",
String.format("%s:8025", resourceManager.getPublicHostName()));
config.setProperty("yarn.app.mapreduce.am.staging-dir", "/user");
- config.setProperty("yarn.application.classpath",
- "/etc/hadoop/conf.dist," +
- "/usr/lib/hadoop/*," +
- "/usr/lib/hadoop/lib/*," +
- "$HADOOP_CONF_DIR," +
- "$HADOOP_COMMON_HOME/share/hadoop/common/*," +
- "$HADOOP_COMMON_HOME/share/hadoop/common/lib/*," +
- "$HADOOP_HDFS_HOME/share/hadoop/hdfs/*," +
- "$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*," +
- "$YARN_HOME/share/hadoop/mapreduce/*," +
- "$YARN_HOME/share/hadoop/mapreduce/lib/*");
return config;
}
Modified:
whirr/trunk/services/yarn/src/main/resources/functions/configure_hadoop_mr2.sh
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/yarn/src/main/resources/functions/configure_hadoop_mr2.sh?rev=1374376&r1=1374375&r2=1374376&view=diff
==============================================================================
---
whirr/trunk/services/yarn/src/main/resources/functions/configure_hadoop_mr2.sh
(original)
+++
whirr/trunk/services/yarn/src/main/resources/functions/configure_hadoop_mr2.sh
Fri Aug 17 17:38:31 2012
@@ -118,6 +118,10 @@ function start_namenode() {
# It's needed to allow users to create their own user directories
$AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -chmod +w /user"
+ # Create job history directory (see
mapreduce.jobhistory.intermediate-done-dir)
+ $AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -mkdir /user/history"
+ $AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -chmod -R 1777 /user/history"
+
# Create temporary directory for Pig and Hive in HDFS
$AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -mkdir /tmp"
$AS_HADOOP "$HADOOP_HOME/bin/hadoop fs -chmod +w /tmp"
Modified:
whirr/trunk/services/yarn/src/main/resources/whirr-yarn-default.properties
URL:
http://svn.apache.org/viewvc/whirr/trunk/services/yarn/src/main/resources/whirr-yarn-default.properties?rev=1374376&r1=1374375&r2=1374376&view=diff
==============================================================================
--- whirr/trunk/services/yarn/src/main/resources/whirr-yarn-default.properties
(original)
+++ whirr/trunk/services/yarn/src/main/resources/whirr-yarn-default.properties
Fri Aug 17 17:38:31 2012
@@ -26,6 +26,7 @@ hadoop-yarn.yarn.nodemanager.remote-app-
hadoop-yarn.yarn.nodemanager.aux-services=mapreduce.shuffle
hadoop-yarn.yarn.nodemanager.aux-services.mapreduce.shuffle.class=org.apache.hadoop.mapred.ShuffleHandler
+hadoop-yarn.yarn.nodemanager.delete.debug-delay-sec=6000
hadoop-yarn.yarn.app.mapreduce.am.staging-dir=/user
