Author: frankscholten
Date: Wed Mar 14 19:25:29 2012
New Revision: 1300692
URL: http://svn.apache.org/viewvc?rev=1300692&view=rev
Log:
WHIRR-335. Update Hama service to Hama 0.4-incubating
Modified:
whirr/branches/branch-0.7/CHANGES.txt
whirr/branches/branch-0.7/pom.xml
whirr/branches/branch-0.7/services/hama/pom.xml
whirr/branches/branch-0.7/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java
whirr/branches/branch-0.7/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java
whirr/branches/branch-0.7/services/hama/src/main/resources/functions/configure_hama.sh
whirr/branches/branch-0.7/services/hama/src/main/resources/functions/install_hama.sh
whirr/branches/branch-0.7/services/hama/src/main/resources/functions/start_hama.sh
whirr/branches/branch-0.7/services/hama/src/main/resources/whirr-hama-default.properties
whirr/branches/branch-0.7/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceTest.java
whirr/branches/branch-0.7/services/hama/src/test/resources/whirr-hama-test.properties
Modified: whirr/branches/branch-0.7/CHANGES.txt
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/CHANGES.txt?rev=1300692&r1=1300691&r2=1300692&view=diff
==============================================================================
--- whirr/branches/branch-0.7/CHANGES.txt (original)
+++ whirr/branches/branch-0.7/CHANGES.txt Wed Mar 14 19:25:29 2012
@@ -21,6 +21,8 @@ Release 0.7.2
WHIRR-553. Upgrade Cassandra to 1.0.8 (Andrei Savu)
+ WHIRR-335. Update Hama service to Hama 0.4-incubating (Edward J. Yoon via
Frank Scholten)
+
BUG FIXES
Release 0.7.1 - 2012-02-23
Modified: whirr/branches/branch-0.7/pom.xml
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/pom.xml?rev=1300692&r1=1300691&r2=1300692&view=diff
==============================================================================
--- whirr/branches/branch-0.7/pom.xml (original)
+++ whirr/branches/branch-0.7/pom.xml Wed Mar 14 19:25:29 2012
@@ -236,7 +236,7 @@
<dependency>
<groupId>org.apache.whirr</groupId>
<artifactId>whirr-build-tools</artifactId>
- <version>${version}</version>
+ <version>${project.version}</version>
</dependency>
</dependencies>
<executions>
Modified: whirr/branches/branch-0.7/services/hama/pom.xml
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hama/pom.xml?rev=1300692&r1=1300691&r2=1300692&view=diff
==============================================================================
--- whirr/branches/branch-0.7/services/hama/pom.xml (original)
+++ whirr/branches/branch-0.7/services/hama/pom.xml Wed Mar 14 19:25:29 2012
@@ -30,7 +30,7 @@
<name>Apache Whirr Hama</name>
<properties>
<hadoop.version>0.20.2</hadoop.version>
- <hama.version>0.3.0-incubating</hama.version>
+ <hama.version>0.4.0-incubating</hama.version>
</properties>
<dependencies>
<dependency>
@@ -116,7 +116,7 @@
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
- <version>3.3.1</version>
+ <version>3.3.3</version>
<scope>test</scope>
</dependency>
</dependencies>
Modified:
whirr/branches/branch-0.7/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java?rev=1300692&r1=1300691&r2=1300692&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java
(original)
+++
whirr/branches/branch-0.7/services/hama/src/main/java/org/apache/whirr/service/hama/HamaGroomServerClusterActionHandler.java
Wed Mar 14 19:25:29 2012
@@ -51,7 +51,7 @@ public class HamaGroomServerClusterActio
InetAddress masterPublicAddress = instance.getPublicAddress();
event.getFirewallManager().addRules(
- Rule.create().destination(instance).ports(61000, GROOMSERVER_PORT));
+ Rule.create().destination(instance).ports(GROOMSERVER_PORT));
String hamaConfigureFunction = getConfiguration(clusterSpec).getString(
HamaConstants.KEY_CONFIGURE_FUNCTION,
Modified:
whirr/branches/branch-0.7/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java?rev=1300692&r1=1300691&r2=1300692&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java
(original)
+++
whirr/branches/branch-0.7/services/hama/src/main/java/org/apache/whirr/service/hama/HamaMasterClusterActionHandler.java
Wed Mar 14 19:25:29 2012
@@ -117,6 +117,8 @@ public class HamaMasterClusterActionHand
config.setProperty("bsp.master.address", master.getHostName() + ":"
+ MASTER_PORT);
+ config.setProperty("fs.default.name", "hdfs://" + master.getHostName()
+ + ":8020");
config.setProperty("hadoop.socks.server", "localhost:6666");
config.setProperty("hadoop.rpc.socket.factory.class.default",
"org.apache.hadoop.net.SocksSocketFactory");
Modified:
whirr/branches/branch-0.7/services/hama/src/main/resources/functions/configure_hama.sh
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hama/src/main/resources/functions/configure_hama.sh?rev=1300692&r1=1300691&r2=1300692&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/hama/src/main/resources/functions/configure_hama.sh
(original)
+++
whirr/branches/branch-0.7/services/hama/src/main/resources/functions/configure_hama.sh
Wed Mar 14 19:25:29 2012
@@ -41,7 +41,7 @@ function configure_hama() {
done
# assign default URL if no other given (optional)
-
HAMA_TAR_URL=${HAMA_TAR_URL:-http://archive.apache.org/dist/incubator/hama/0.3-incubating/hama-0.3.0-incubating.tar.gz}
+
HAMA_TAR_URL=${HAMA_TAR_URL:-http://archive.apache.org/dist/incubator/hama/0.4-incubating/hama-0.4.0-incubating.tar.gz}
# derive details from the URL
HAMA_TAR_FILE=${HAMA_TAR_URL##*/}
Modified:
whirr/branches/branch-0.7/services/hama/src/main/resources/functions/install_hama.sh
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hama/src/main/resources/functions/install_hama.sh?rev=1300692&r1=1300691&r2=1300692&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/hama/src/main/resources/functions/install_hama.sh
(original)
+++
whirr/branches/branch-0.7/services/hama/src/main/resources/functions/install_hama.sh
Wed Mar 14 19:25:29 2012
@@ -36,7 +36,7 @@ function install_hama() {
done
# assign default URL if no other given (optional)
-
HAMA_TAR_URL=${HAMA_TAR_URL:-http://archive.apache.org/dist/incubator/hama/0.3-incubating/hama-0.3.0-incubating.tar.gz}
+
HAMA_TAR_URL=${HAMA_TAR_URL:-http://archive.apache.org/dist/incubator/hama/0.4-incubating/hama-0.4.0-incubating.tar.gz}
# derive details from the URL
HAMA_TAR_FILE=${HAMA_TAR_URL##*/}
HAMA_TAR_MD5_FILE=$HAMA_TAR_FILE.md5
Modified:
whirr/branches/branch-0.7/services/hama/src/main/resources/functions/start_hama.sh
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hama/src/main/resources/functions/start_hama.sh?rev=1300692&r1=1300691&r2=1300692&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/hama/src/main/resources/functions/start_hama.sh
(original)
+++
whirr/branches/branch-0.7/services/hama/src/main/resources/functions/start_hama.sh
Wed Mar 14 19:25:29 2012
@@ -34,7 +34,7 @@ function start_hama() {
done
# assign default URL if no other given (optional)
-
HAMA_TAR_URL=${HAMA_TAR_URL:-http://archive.apache.org/dist/incubator/hama/0.3-incubating/hama-0.3.0-incubating.tar.gz}
+
HAMA_TAR_URL=${HAMA_TAR_URL:-http://archive.apache.org/dist/incubator/hama/0.4-incubating/hama-0.4.0-incubating.tar.gz}
# derive details from the URL
HAMA_TAR_FILE=${HAMA_TAR_URL##*/}
Modified:
whirr/branches/branch-0.7/services/hama/src/main/resources/whirr-hama-default.properties
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hama/src/main/resources/whirr-hama-default.properties?rev=1300692&r1=1300691&r2=1300692&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/hama/src/main/resources/whirr-hama-default.properties
(original)
+++
whirr/branches/branch-0.7/services/hama/src/main/resources/whirr-hama-default.properties
Wed Mar 14 19:25:29 2012
@@ -10,4 +10,5 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-whirr.hama.tarball.url=http://archive.apache.org/dist/incubator/hama/0.3-incubating/hama-0.3.0-incubating.tar.gz
+whirr.hadoop.version=0.20.2
+whirr.hama.tarball.url=http://archive.apache.org/dist/incubator/hama/0.4-incubating/hama-0.4.0-incubating.tar.gz
Modified:
whirr/branches/branch-0.7/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceTest.java
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceTest.java?rev=1300692&r1=1300691&r2=1300692&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceTest.java
(original)
+++
whirr/branches/branch-0.7/services/hama/src/test/java/org/apache/whirr/service/hama/integration/HamaServiceTest.java
Wed Mar 14 19:25:29 2012
@@ -17,25 +17,38 @@
*/
package org.apache.whirr.service.hama.integration;
-import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.DoubleWritable;
+import org.apache.hadoop.io.Text;
import org.apache.hama.HamaConfiguration;
import org.apache.hama.bsp.BSPJob;
import org.apache.hama.bsp.BSPJobClient;
+import org.apache.hama.bsp.BSPJobID;
import org.apache.hama.bsp.ClusterStatus;
-import org.apache.hama.examples.PiEstimator;
+import org.apache.hama.bsp.FileOutputFormat;
+import org.apache.hama.bsp.NullInputFormat;
+import org.apache.hama.bsp.RunningJob;
+import org.apache.hama.bsp.TextOutputFormat;
import org.apache.hama.examples.PiEstimator.MyEstimator;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class HamaServiceTest {
+ private static final Logger LOG = LoggerFactory
+ .getLogger(HamaServiceTest.class);
+
private static HamaServiceController controller = HamaServiceController
.getInstance();
- private static String masterTask = "master.task.";
-
+
+ private final static Path TMP_OUTPUT = new Path("/pi-"
+ + System.currentTimeMillis());
+
@BeforeClass
public static void setUp() throws Exception {
controller.ensureClusterRunning();
@@ -48,27 +61,40 @@ public class HamaServiceTest {
@Test
public void test() throws Exception {
- HamaConfiguration conf = controller.getConfiguration();
+ HamaConfiguration jobConf = controller.getConfiguration();
+ jobConf.set("hadoop.rpc.socket.factory.class.default",
+ "org.apache.hadoop.net.StandardSocketFactory");
+
+ BSPJob bsp = new BSPJob(jobConf, new BSPJobID());
+ LOG.info("Job conf: "
+ + bsp.getConf().get("hadoop.rpc.socket.factory.class.default") + ", "
+ + bsp.getJobID().toString());
- BSPJobClient jobClient = new BSPJobClient(conf);
+ bsp.setJarByClass(MyEstimator.class);
+ bsp.setBspClass(MyEstimator.class);
+ bsp.setInputFormat(NullInputFormat.class);
+ bsp.setOutputKeyClass(Text.class);
+ bsp.setOutputValueClass(DoubleWritable.class);
+ bsp.setOutputFormat(TextOutputFormat.class);
+ bsp.set("bsp.working.dir", "/tmp");
+ FileOutputFormat.setOutputPath(bsp, TMP_OUTPUT);
+
+ LOG.info("Client configuration start ..");
+
+ HamaConfiguration clientConf = controller.getConfiguration();
+ BSPJobClient jobClient = new BSPJobClient(clientConf);
ClusterStatus cluster = jobClient.getClusterStatus(true);
assertNotNull(cluster);
assertTrue(cluster.getGroomServers() > 0);
+ assertTrue(cluster.getMaxTasks() > 1);
+ bsp.setNumBspTask(cluster.getMaxTasks());
- BSPJob bsp = new BSPJob(conf, PiEstimator.class);
- // Set the job name
- bsp.setJobName("Pi Estimation Example");
- bsp.setBspClass(MyEstimator.class);
- bsp.setNumBspTask(cluster.getGroomServers());
+ LOG.info("Client conf: "
+ + clientConf.get("hadoop.rpc.socket.factory.class.default"));
- for (String peerName : cluster.getActiveGroomNames().values()) {
- conf.set(masterTask, peerName);
- break;
- }
-
- if (bsp.waitForCompletion(true)) {
- assertEquals(jobClient.getAllJobs().length, 1);
- }
+ RunningJob rJob = jobClient.submitJob(bsp);
+ rJob.waitForCompletion();
+ LOG.info("finished");
}
}
Modified:
whirr/branches/branch-0.7/services/hama/src/test/resources/whirr-hama-test.properties
URL:
http://svn.apache.org/viewvc/whirr/branches/branch-0.7/services/hama/src/test/resources/whirr-hama-test.properties?rev=1300692&r1=1300691&r2=1300692&view=diff
==============================================================================
---
whirr/branches/branch-0.7/services/hama/src/test/resources/whirr-hama-test.properties
(original)
+++
whirr/branches/branch-0.7/services/hama/src/test/resources/whirr-hama-test.properties
Wed Mar 14 19:25:29 2012
@@ -15,6 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+whirr.hadoop.version=0.20.2
whirr.cluster-name=hama-itest-${sys:whirr.test.provider}-${sys:user.name}
whirr.instance-templates=1 zookeeper,1
hadoop-namenode+hadoop-jobtracker+hama-master,1
hadoop-datanode+hadoop-tasktracker+hama-groomserver
@@ -22,5 +23,5 @@ whirr.provider=${sys:whirr.test.provider
whirr.identity=${sys:whirr.test.identity}
whirr.credential=${sys:whirr.test.credential}
-whirr.hama.tarball.url=http://archive.apache.org/dist/incubator/hama/0.3-incubating/hama-0.3.0-incubating.tar.gz
+whirr.hama.tarball.url=http://archive.apache.org/dist/incubator/hama/0.4-incubating/hama-0.4.0-incubating.tar.gz
