Repository: bigtop Updated Branches: refs/heads/master f69597dc2 -> bfa820006
BIGTOP-2317. Add smoke tests for QFS Signed-off-by: Konstantin Boudnik <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/bfa82000 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/bfa82000 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/bfa82000 Branch: refs/heads/master Commit: bfa820006204ddb68160a3132958e1478c2d681e Parents: f69597d Author: Faraaz Sareshwala <[email protected]> Authored: Thu Feb 11 11:53:44 2016 -0800 Committer: Konstantin Boudnik <[email protected]> Committed: Sat Apr 30 17:38:18 2016 -0700 ---------------------------------------------------------------------- bigtop-deploy/vm/utils/smoke-tests.sh | 14 ++++-- bigtop-tests/smoke-tests/qfs/build.gradle | 45 ++++++++++++++++++++ .../hadoop/mapreduce/TestHadoopExamples.groovy | 19 ++++++--- .../hadoop/mapreduce/TestHadoopSmoke.groovy | 17 +++++--- 4 files changed, 79 insertions(+), 16 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/bfa82000/bigtop-deploy/vm/utils/smoke-tests.sh ---------------------------------------------------------------------- diff --git a/bigtop-deploy/vm/utils/smoke-tests.sh b/bigtop-deploy/vm/utils/smoke-tests.sh index 8dac31c..93f795e 100755 --- a/bigtop-deploy/vm/utils/smoke-tests.sh +++ b/bigtop-deploy/vm/utils/smoke-tests.sh @@ -36,9 +36,17 @@ export SQOOP_HOME=/usr/lib/sqoop/ export HIVE_CONF_DIR=/etc/hive/conf/ export MAHOUT_HOME="/usr/lib/mahout" -su -s /bin/bash $HCFS_USER -c '/usr/bin/hadoop fs -mkdir /user/vagrant /user/root' -su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/vagrant' -su -s /bin/bash $HCFS_USER -c 'hadoop fs -chmod 777 /user/root' +prep() { + HADOOP_COMMAND=$1 + su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -mkdir /user/vagrant /user/root" + su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/vagrant" + su -s /bin/bash $HCFS_USER -c "JAVA_LIBRARY_PATH=/usr/lib/qfs $HADOOP_COMMAND fs -chmod 777 /user/root" +} + +prep hadoop +if [[ $SMOKE_TESTS == *"qfs"* ]]; then + prep hadoop-qfs +fi if [ -f /etc/debian_version ] ; then apt-get -y install pig hive flume mahout sqoop http://git-wip-us.apache.org/repos/asf/bigtop/blob/bfa82000/bigtop-tests/smoke-tests/qfs/build.gradle ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/qfs/build.gradle b/bigtop-tests/smoke-tests/qfs/build.gradle new file mode 100644 index 0000000..ca25635 --- /dev/null +++ b/bigtop-tests/smoke-tests/qfs/build.gradle @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +def tests_to_include() { + def smoke_tests = System.getenv('SMOKE_TESTS') + if (smoke_tests == null || !smoke_tests.contains("qfs")) { + return []; + } + + return [ + "TestHadoopExamples.groovy" + ]; +} + +sourceSets { + test { + groovy { + srcDirs = [ + "${BIGTOP_HOME}/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce" + ] + exclude { + FileTreeElement elem -> (doExclude(elem.getName())) + } + } + } +} + +test.doFirst { + environment("HADOOP_COMMAND", "hadoop-qfs") + checkEnv(["HADOOP_CONF_DIR", "HADOOP_MAPRED_HOME"]) +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/bfa82000/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy index 5a19945..77af19c 100644 --- a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy +++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy @@ -42,6 +42,11 @@ class TestHadoopExamples { private static final String HADOOP_MAPRED_HOME = System.getenv('HADOOP_MAPRED_HOME'); private static final String HADOOP_CONF_DIR = System.getenv('HADOOP_CONF_DIR'); + + // The hadoop command is dynamic in order to support both hadoop over hdfs + // and hadoop over qfs easily. + private static final String HADOOP_COMMAND = System.getProperty('HADOOP_COMMAND', 'hadoop'); + private static String hadoopExamplesJar = JarContent.getJarName(HADOOP_MAPRED_HOME, 'hadoop.*examples.*.jar'); static { @@ -73,8 +78,8 @@ class TestHadoopExamples { @AfterClass public static void tearDown() { - sh.exec("hadoop fs -rmr -skipTrash ${EXAMPLES}", - "hadoop fs -rmr -skipTrash ${EXAMPLES_OUT}"); + sh.exec("${HADOOP_COMMAND} fs -rmr -skipTrash ${EXAMPLES}", + "${HADOOP_COMMAND} fs -rmr -skipTrash ${EXAMPLES_OUT}"); } @@ -96,12 +101,12 @@ class TestHadoopExamples { LOG.info("MAKING DIRECTORIES ..................... ${EXAMPLES} ${EXAMPLES_OUT}"); //add the files in resources/ - sh.exec("hadoop fs -put ${source}/*.* ."); + sh.exec("${HADOOP_COMMAND} fs -put ${source}/*.* ."); //add the directories under resources (like examples/) - sh.exec("hadoop fs -put ${source}/${EXAMPLES} ${EXAMPLES}"); - sh.exec("hadoop fs -mkdir -p ${EXAMPLES_OUT}"); + sh.exec("${HADOOP_COMMAND} fs -put ${source}/${EXAMPLES} ${EXAMPLES}"); + sh.exec("${HADOOP_COMMAND} fs -mkdir -p ${EXAMPLES_OUT}"); } - sh.exec("hadoop fs -ls ${EXAMPLES}"); + sh.exec("${HADOOP_COMMAND} fs -ls ${EXAMPLES}"); assertTrue("Failed asserting that 'examples' were created in the DFS", sh.getRet() == 0); } @@ -153,7 +158,7 @@ class TestHadoopExamples { || FailureVars.instance.getNetworkShutdown()) { runFailureThread(); } - sh.exec("hadoop jar $testJar $testName $testArgs"); + sh.exec("${HADOOP_COMMAND} jar $testJar $testName $testArgs"); assertTrue("Example $testName $testJar $testName $testArgs failed", sh.getRet() == 0); } http://git-wip-us.apache.org/repos/asf/bigtop/blob/bfa82000/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy index 63998ac..118bbb6 100644 --- a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy +++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy @@ -36,6 +36,11 @@ class TestHadoopSmoke { static String hadoopHome = System.getProperty('HADOOP_HOME', '/usr/lib/hadoop') static String streamingHome = System.getenv('STREAMING_HOME') static String hadoopMapReduceHome = System.getProperty('HADOOP_MAPRED_HOME', '/usr/lib/hadoop-mapreduce') + + // The hadoop command is dynamic in order to support both hadoop over hdfs + // and hadoop over qfs easily. + private static final String HADOOP_COMMAND = System.getProperty('HADOOP_COMMAND', 'hadoop'); + static final String STREAMING_HOME = (streamingHome == null) ? hadoopMapReduceHome : streamingHome; static String streaming_jar = @@ -47,7 +52,7 @@ class TestHadoopSmoke { static String testDir = "test.hadoopsmoke." + (new Date().getTime()) static String nn = (new Configuration()).get(DFSConfigKeys.FS_DEFAULT_NAME_KEY) - String cmd = "hadoop jar ${STREAMING_JAR}" + + String cmd = "${HADOOP_COMMAND} jar ${STREAMING_JAR}" + " -D mapred.map.tasks=1 -D mapred.reduce.tasks=1 -D mapred.job.name=Experiment" String cmd2 = " -input ${testDir}/cachefile/input.txt -mapper map.sh -file map.sh -reducer cat" + " -output ${testDir}/cachefile/out -verbose" @@ -65,15 +70,15 @@ class TestHadoopSmoke { @AfterClass static void tearDown() { - sh.exec("hadoop fs -rmr -skipTrash ${testDir}") + sh.exec("${HADOOP_COMMAND} fs -rmr -skipTrash ${testDir}") } @Test (timeout = 0x810000l) void testCacheArchive() { - sh.exec("hadoop fs -rmr ${testDir}/cachefile/out", + sh.exec("${HADOOP_COMMAND} fs -rmr ${testDir}/cachefile/out", cmd + ' -cacheArchive ' + arg + cmd2) logError(sh) - sh.exec("hadoop fs -cat ${testDir}/cachefile/out/part-00000") + sh.exec("${HADOOP_COMMAND} fs -cat ${testDir}/cachefile/out/part-00000") logError(sh) assertEquals("cache1\t\ncache2\t", sh.out.join('\n')) @@ -81,10 +86,10 @@ class TestHadoopSmoke { @Test (timeout = 0x810000l) void testArchives() { - sh.exec("hadoop fs -rmr ${testDir}/cachefile/out", + sh.exec("${HADOOP_COMMAND} fs -rmr ${testDir}/cachefile/out", cmd + ' -archives ' + arg + cmd2) logError(sh) - sh.exec("hadoop fs -cat ${testDir}/cachefile/out/part-00000") + sh.exec("${HADOOP_COMMAND} fs -cat ${testDir}/cachefile/out/part-00000") logError(sh) assertEquals("cache1\t\ncache2\t", sh.out.join('\n'))
