BIGTOP-956. fix small issues with mahout, hue and oozie tests
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/a4eb310b Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/a4eb310b Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/a4eb310b Branch: refs/heads/master Commit: a4eb310b236314379aed4a82036d01fa8b51a992 Parents: 3414748 Author: Roman Shaposhnik <[email protected]> Authored: Wed May 1 11:05:51 2013 -0700 Committer: Roman Shaposhnik <[email protected]> Committed: Wed May 1 14:50:38 2013 -0700 ---------------------------------------------------------------------- .../bigtop/itest/huesmoke/TestHueSmoke.groovy | 4 +- .../itest/mahout/smoke/TestMahoutExamples.groovy | 69 ++++++++------- .../bigtop/itest/ooziesmoke/TestOozieSmoke.groovy | 10 -- bigtop-tests/test-execution/smokes/hue/pom.xml | 34 +++++++- 4 files changed, 72 insertions(+), 45 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/a4eb310b/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy b/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy index da12d50..50c0a46 100644 --- a/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy +++ b/bigtop-tests/test-artifacts/hue/src/main/groovy/org/apache/bigtop/itest/huesmoke/TestHueSmoke.groovy @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.bigtop.itest.giraph.smoke +package org.apache.bigtop.itest.hue.smoke import static org.junit.Assert.assertEquals import static org.junit.Assert.assertTrue @@ -25,7 +25,7 @@ import org.apache.bigtop.itest.shell.Shell import org.junit.Test public class TestHueSmoke { - final static String hueServer = "http://localhost:8888"; + final static String hueServer = System.getProperty("org.apache.bigtop.itest.hue_url", "http://localhost:8888";); final static String loginURL = "${hueServer}/accounts/login/"; final static String checkURL = "${hueServer}/debug/check_config"; final static String creds = "username=admin&password=admin"; http://git-wip-us.apache.org/repos/asf/bigtop/blob/a4eb310b/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy b/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy index 586792c..8cc8efc 100644 --- a/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy +++ b/bigtop-tests/test-artifacts/mahout/src/main/groovy/org/apache/bigtop/itest/mahout/smoke/TestMahoutExamples.groovy @@ -102,8 +102,12 @@ public class TestMahoutExamples { } } - @Test(timeout=1200000L) + @Test(timeout=12000000L) public void factorizeMovieLensRatings() { + // convert ratings + sh.exec("cat ${TEMP_DIR}/movielens/ml-1m/ratings.dat |sed -e s/::/,/g| cut -d, -f1,2,3 > ${TEMP_DIR}/movielens/ratings.csv"); + assertEquals("Unexpected error from converting ratings", 0, sh.getRet()); + // put ratings in hdfs sh.exec("hadoop fs -mkdir ${WORK_DIR}/movielens", "hadoop fs -put ${TEMP_DIR}/movielens/ratings.csv ${WORK_DIR}/movielens/ratings.csv"); @@ -218,40 +222,43 @@ public class TestMahoutExamples { @Test(timeout=9000000L) public void testBayesNewsgroupClassifier() { - sh.exec("""mahout org.apache.mahout.classifier.bayes.PrepareTwentyNewsgroups \ - -p ${TEMP_DIR}/20news-bydate/20news-bydate-train \ - -o ${TEMP_DIR}/20news-bydate/bayes-train-input \ - -a org.apache.mahout.vectorizer.DefaultAnalyzer \ - -c UTF-8"""); + // put bayes-train-input and bayes-test-input in hdfs + sh.exec("hadoop fs -mkdir ${WORK_DIR}/20news-vectors"); + sh.exec("hadoop fs -put ${TEMP_DIR}/20news-all ${WORK_DIR}/20news-all"); + assertEquals("Unable to put bayes-train-input in hdfs", 0, sh.getRet()); + + sh.exec("mahout seqdirectory -i ${WORK_DIR}/20news-all -o ${WORK_DIR}/20news-seq"); assertEquals("Unexpected error from running mahout", 0, sh.getRet()); - sh.exec("""mahout org.apache.mahout.classifier.bayes.PrepareTwentyNewsgroups \ - -p ${TEMP_DIR}/20news-bydate/20news-bydate-test \ - -o ${TEMP_DIR}/20news-bydate/bayes-test-input \ - -a org.apache.mahout.vectorizer.DefaultAnalyzer \ - -c UTF-8"""); + + sh.exec("mahout seq2sparse -i ${WORK_DIR}/20news-seq -o ${WORK_DIR}/20news-vectors -lnorm -nv -wt tfidf"); assertEquals("Unexpected error from running mahout", 0, sh.getRet()); - // put bayes-train-input and bayes-test-input in hdfs - sh.exec("hadoop fs -mkdir ${WORK_DIR}/20news-bydate"); - sh.exec("hadoop fs -put ${TEMP_DIR}/20news-bydate/bayes-train-input ${WORK_DIR}/20news-bydate/bayes-train-input"); - assertEquals("Unable to put bayes-train-input in hdfs", 0, sh.getRet()); - sh.exec("hadoop fs -put ${TEMP_DIR}/20news-bydate/bayes-test-input ${WORK_DIR}/20news-bydate/bayes-test-input"); - assertEquals("Unable to put bayes-test-input in hdfs", 0, sh.getRet()); - - sh.exec("""mahout trainclassifier \ --i ${WORK_DIR}/20news-bydate/bayes-train-input \ --o ${WORK_DIR}/20news-bydate/bayes-model \ --type bayes \ --ng 1 \ --source hdfs"""); + sh.exec("""mahout split \ +-i ${WORK_DIR}/20news-vectors/tfidf-vectors \ +--trainingOutput ${WORK_DIR}/20news-train-vectors \ +--testOutput ${WORK_DIR}/20news-test-vectors \ +--randomSelectionPct 40 --overwrite --sequenceFiles -xm sequential"""); + assertEquals("Unexpected error from running mahout", 0, sh.getRet()); + + sh.exec("""mahout trainnb \ +-i ${WORK_DIR}/20news-train-vectors -el \ +-o ${WORK_DIR}/model \ +-li ${WORK_DIR}/labelindex \ +-ow"""); assertEquals("Unexpected error from running mahout", 0, sh.getRet()); - sh.exec("""mahout testclassifier \ --m ${WORK_DIR}/20news-bydate/bayes-model \ --d ${WORK_DIR}/20news-bydate/bayes-test-input \ --type bayes \ --ng 1 \ --source hdfs \ --method mapreduce"""); + + sh.exec("""mahout testnb \ +-i ${WORK_DIR}/20news-train-vectors \ +-m ${WORK_DIR}/model \ +-l ${WORK_DIR}/labelindex \ +-ow -o ${WORK_DIR}/20news-testing"""); + assertEquals("Unexpected error from running mahout", 0, sh.getRet()); + + sh.exec("""mahout testnb \ +-i ${WORK_DIR}/20news-test-vectors \ +-m ${WORK_DIR}/model \ +-l ${WORK_DIR}/labelindex \ +-ow -o ${WORK_DIR}/20news-testing"""); assertEquals("Unexpected error from running mahout", 0, sh.getRet()); } http://git-wip-us.apache.org/repos/asf/bigtop/blob/a4eb310b/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy b/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy index 23a00cb..7d1a90c 100644 --- a/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy +++ b/bigtop-tests/test-artifacts/oozie/src/main/groovy/org/apache/bigtop/itest/ooziesmoke/TestOozieSmoke.groovy @@ -125,16 +125,6 @@ class TestOozieSmoke { } @Test(timeout=300000L) - public void testSqoop() { - testOozieExamplesCommon("sqoop"); - } - - @Test(timeout=300000L) - public void testSqoopFreeform() { - testOozieExamplesCommon("sqoop-freeform"); - } - - @Test(timeout=300000L) public void testSubwf() { testOozieExamplesCommon("subwf"); } http://git-wip-us.apache.org/repos/asf/bigtop/blob/a4eb310b/bigtop-tests/test-execution/smokes/hue/pom.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-execution/smokes/hue/pom.xml b/bigtop-tests/test-execution/smokes/hue/pom.xml index ed96ae7..19e27a6 100644 --- a/bigtop-tests/test-execution/smokes/hue/pom.xml +++ b/bigtop-tests/test-execution/smokes/hue/pom.xml @@ -30,9 +30,11 @@ <groupId>org.apache.bigtop.itest</groupId> <artifactId>hue-smoke-execution</artifactId> <version>0.6.0-SNAPSHOT</version> - <name>Giraph smoke test execution</name> + <name>Hue smoke test execution</name> <properties> + <HUE_URL>${env.HUE_URL}</HUE_URL> + <org.apache.maven-dependency-plugin.groupId>org.apache.bigtop.itest</org.apache.maven-dependency-plugin.groupId> <org.apache.maven-dependency-plugin.artifactId>hue-smoke</org.apache.maven-dependency-plugin.artifactId> <org.apache.maven-dependency-plugin.version>${hue-smoke.version}</org.apache.maven-dependency-plugin.version> @@ -55,8 +57,36 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-failsafe-plugin</artifactId> + <configuration> + <systemPropertyVariables> + <org.apache.bigtop.itest.hue_url>${HUE_URL}</org.apache.bigtop.itest.hue_url> + </systemPropertyVariables> + <forkMode>always</forkMode> + </configuration> + </plugin> + + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-enforcer-plugin</artifactId> + <version>1.0</version> + <executions> + <execution> + <id>enforce-property-oozie</id> + <goals> + <goal>enforce</goal> + </goals> + <configuration> + <rules> + <requireProperty> + <property>HUE_URL</property> + <message>HUE_URL env. variable has to be set</message> + </requireProperty> + </rules> + <fail>true</fail> + </configuration> + </execution> + </executions> </plugin> </plugins> </build> - </project>
