BIGTOP-793 add crunch integration tests
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/4a307414 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/4a307414 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/4a307414 Branch: refs/heads/master Commit: 4a3074149297ecb188d55bf05d8b4a751a7b92d1 Parents: 57fbb28 Author: Johnny Zhang <[email protected]> Authored: Thu Nov 29 19:53:37 2012 -0500 Committer: Roman Shaposhnik <[email protected]> Committed: Tue Dec 4 13:59:46 2012 -0800 ---------------------------------------------------------------------- bigtop-tests/test-artifacts/crunch/pom.xml | 32 + .../itest/crunchsmoke/TestCrunchSmoke.groovy | 92 + .../resources/crunch-examples/access_log/000000 |13501 +++++++++++++++ .../main/resources/crunch-examples/text/pg11.txt | 3735 ++++ .../crunch-examples/text/secondary_sort_input.txt | 8 + bigtop-tests/test-execution/smokes/crunch/pom.xml | 98 + bigtop-tests/test-execution/smokes/pom.xml | 2 + 7 files changed, 17468 insertions(+), 0 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a307414/bigtop-tests/test-artifacts/crunch/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/crunch/pom.xml b/bigtop-tests/test-artifacts/crunch/pom.xml new file mode 100644 index 0000000..80b069e --- /dev/null +++ b/bigtop-tests/test-artifacts/crunch/pom.xml @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>bigtop-smokes</artifactId> + <version>0.5.0-SNAPSHOT</version> + <relativePath>../pom.xml</relativePath> + </parent> + + <groupId>org.apache.bigtop.itest</groupId> + <artifactId>crunch-smoke</artifactId> + <version>0.5.0-SNAPSHOT</version> + <name>crunchsmoke</name> +</project> http://git-wip-us.apache.org/repos/asf/bigtop/blob/4a307414/bigtop-tests/test-artifacts/crunch/src/main/groovy/org/apache/bigtop/itest/crunchsmoke/TestCrunchSmoke.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/crunch/src/main/groovy/org/apache/bigtop/itest/crunchsmoke/TestCrunchSmoke.groovy b/bigtop-tests/test-artifacts/crunch/src/main/groovy/org/apache/bigtop/itest/crunchsmoke/TestCrunchSmoke.groovy new file mode 100644 index 0000000..5a7a931 --- /dev/null +++ b/bigtop-tests/test-artifacts/crunch/src/main/groovy/org/apache/bigtop/itest/crunchsmoke/TestCrunchSmoke.groovy @@ -0,0 +1,92 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.itest.crunch.smoke + +import org.junit.BeforeClass +import org.junit.Test +import static org.junit.Assert.assertEquals +import static org.junit.Assert.assertTrue + +import org.apache.bigtop.itest.shell.Shell + +public class TestCrunchSmoke { + static String runnerScript = "hadoop jar" + + static String crunchJar = "/usr/lib/crunch/crunch-examples-*job.jar"; + + static Shell sh = new Shell("/bin/bash -s"); + private static final String EXAMPLES = "crunch-examples"; + private static final String EXAMPLES_OUT = "crunch-examples-output"; + + @BeforeClass + static void setUp() { + sh.exec("hadoop fs -test -e $EXAMPLES"); + if (sh.getRet() == 0) { + sh.exec("hadoop fs -rmr -skipTrash $EXAMPLES"); + assertTrue("Deletion of previous $EXAMPLES from HDFS failed", + sh.getRet() == 0); + } + sh.exec("hadoop fs -test -e $EXAMPLES_OUT"); + if (sh.getRet() == 0) { + sh.exec("hadoop fs -rmr -skipTrash $EXAMPLES_OUT"); + assertTrue("Deletion of previous examples output from HDFS failed", + sh.getRet() == 0); + } + + // copy test files to HDFS + sh.exec("hadoop fs -put $EXAMPLES $EXAMPLES", + "hadoop fs -mkdir $EXAMPLES_OUT"); + assertTrue("Could not create output directory", sh.getRet() == 0); + } + + static Map examples = + [ + WordCount : "${EXAMPLES}/text/pg11.txt $EXAMPLES_OUT", + SecondarySortExample : "${EXAMPLES}/text/secondary_sort_input.txt ${EXAMPLES_OUT}", + AverageBytesByIP : "${EXAMPLES}/access_log/000000 ${EXAMPLES_OUT}", + TotalBytesByIP : "${EXAMPLES}/access_log/000000 ${EXAMPLES_OUT}" + ]; + + private void _runExampleJobs(String algorithm) { + sh.exec("${runnerScript} ${crunchJar}" + + " org.apache.crunch.examples.${algorithm}" + + " ${examples.get(algorithm)}" + ); + assertEquals("running Crunch example failed", sh.getRet(), 0); + } + + @Test(timeout=300000L) + public void testWordCount() { + _runExampleJobs("WordCount"); + } + + @Test(timeout=300000L) + public void testSecondarySort() { + _runExampleJobs("SecondarySortExample"); + } + + @Test(timeout=300000L) + public void testAverageBytesByIP() { + _runExampleJobs("AverageBytesByIP"); + } + + @Test(timeout=300000L) + public void testTotalBytesByIP() { + _runExampleJobs("TotalBytesByIP"); + } +}
