Updated Branches: refs/heads/branch-0.3.1 2c4f5c6e0 -> 0c8282c23
BIGTOP-731 Reorganize Hadoop tests (by Wing Yew Poon) Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/0c8282c2 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/0c8282c2 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/0c8282c2 Branch: refs/heads/branch-0.3.1 Commit: 0c8282c23bb209e742a7e3c9ea003ea80cd4e261 Parents: 2c4f5c6 Author: Stephen Chu <[email protected]> Authored: Wed Dec 5 13:02:33 2012 -0800 Committer: Stephen Chu <[email protected]> Committed: Wed Dec 5 13:02:33 2012 -0800 ---------------------------------------------------------------------- .../bigtop/itest/hadoop/hdfs/TestTestCLI.groovy | 57 +++++++ .../bigtop/itest/hadoop/hdfs/TestTextSnappy.groovy | 67 ++++++++ .../hadoop/mapreduce/TestHadoopExamples.groovy | 125 +++++++++++++++ .../itest/hadoop/mapreduce/TestHadoopSmoke.groovy | 95 +++++++++++ .../itest/hadoopexamples/TestHadoopExamples.groovy | 125 --------------- .../itest/hadoopsmoke/TestHadoopSmoke.groovy | 95 ----------- .../bigtop/itest/hadooptests/TestTestCLI.groovy | 57 ------- .../bigtop/itest/hadooptests/TestTextSnappy.groovy | 67 -------- 8 files changed, 344 insertions(+), 344 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c8282c2/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestTestCLI.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestTestCLI.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestTestCLI.groovy new file mode 100644 index 0000000..4bbd0b7 --- /dev/null +++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestTestCLI.groovy @@ -0,0 +1,57 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.bigtop.itest.hadoop.hdfs + +import org.junit.Ignore +import org.junit.Test +import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.cli.* + +// TODO: we have to stub it for 0.20.2 release, once we move to 0.21+ this can go +// import org.apache.hadoop.fs.CommonConfigurationKeys +class CommonConfigurationKeys { + static final public String FS_DEFAULT_NAME_KEY = "fs.default.name"; +} + +/** This test class serves only one purpose: to prepare environment for proper + execution of parent (TestCLI) + */ +class TestTestCLI extends TestCLI { + private static final String JT = "mapred.job.tracker"; + private static Configuration conf = new Configuration(); + + static { + conf.addDefaultResource('mapred-site.xml'); + System.properties.setProperty(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, + conf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY)); + System.properties.setProperty(JT, conf.get(JT)); + System.properties.setProperty('test.cli.config', 'testConfCluster.xml'); + // The following doesn't really work because in the parent this property + // is used to set a value to a static member. Basically, by the time + // we get here it's too late already. + // The property needs to be set from runtime + System.properties.setProperty('test.cache.data', 'clitest_data'); + } + + @Test + @Ignore("HADOOP-7730") + @Override + public void testAll() { + super.testAll(); + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c8282c2/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestTextSnappy.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestTextSnappy.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestTextSnappy.groovy new file mode 100644 index 0000000..0b3a62b --- /dev/null +++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/hdfs/TestTextSnappy.groovy @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bigtop.itest.hadoop.hdfs + +import org.apache.bigtop.itest.shell.Shell +import org.junit.AfterClass +import org.junit.BeforeClass +import org.junit.Test +import static org.junit.Assert.assertEquals + +class TestTextSnappy { + static Shell sh = new Shell("/bin/bash -s") + static String testDir = "testtextsnappy." + (new Date().getTime()) + static String snappyFile = "part-00001.snappy" + + @BeforeClass + static void setUp() throws IOException { + sh.exec( + "hadoop fs -mkdir ${testDir}", + "hadoop fs -put ${snappyFile} ${testDir}/${snappyFile}", + ) + logError(sh) + } + + @AfterClass + static void tearDown() { + sh.exec("hadoop fs -rmr -skipTrash ${testDir}") + } + + @Test + void testTextSnappy() { + String cmd = "hadoop fs -text ${testDir}/${snappyFile}" + System.out.println(cmd) + sh.exec(cmd) + String output = sh.getOut().join("\n") + logError(sh) + String expected = "1\trafferty\t31\n2\tjones\t33\n3\tsteinberg\t33" + System.out.println("Expected output:\n${expected}") + System.out.println("Actual output:\n${output}") + assertEquals("Incorrect output", output, expected) + } + + private static void logError (final Shell sh) { + if (sh.getRet()) { + println ('Failed command: ' + sh.script); + println ('\terror code: ' + sh.getRet()); + println ('\tstdout: ' + sh.getOut()); + println ('\tstderr: ' + sh.getErr()); + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c8282c2/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy new file mode 100644 index 0000000..21ee00e --- /dev/null +++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopExamples.groovy @@ -0,0 +1,125 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bigtop.itest.hadoop.mapreduce + +import org.junit.BeforeClass +import static org.junit.Assert.assertNotNull +import org.apache.bigtop.itest.shell.Shell +import static org.junit.Assert.assertTrue +import org.junit.Test +import org.apache.hadoop.conf.Configuration +import org.apache.bigtop.itest.JarContent +import org.apache.commons.logging.LogFactory +import org.apache.commons.logging.Log + +import org.apache.bigtop.itest.junit.OrderedParameterized +import org.junit.runners.Parameterized.Parameters +import org.junit.runner.RunWith + +@RunWith(OrderedParameterized.class) +class TestHadoopExamples { + static private Log LOG = LogFactory.getLog(TestHadoopExamples.class); + + private static final String HADOOP_HOME = System.getenv('HADOOP_HOME'); + private static final String HADOOP_CONF_DIR = System.getenv('HADOOP_CONF_DIR'); + private static String hadoopExamplesJar = + JarContent.getJarName(HADOOP_HOME, 'hadoop.*examples.*.jar'); + static { + assertNotNull("HADOOP_HOME has to be set to run this test", + HADOOP_HOME); + assertNotNull("HADOOP_CONF_DIR has to be set to run this test", + HADOOP_CONF_DIR); + assertNotNull("Can't find hadoop-examples.jar file", hadoopExamplesJar); + } + static final String HADOOP_EXAMPLES_JAR = + HADOOP_HOME + "/" + hadoopExamplesJar; + private static final String hadoop = "$HADOOP_HOME/bin/hadoop"; + + static Shell sh = new Shell("/bin/bash -s"); + private static final String EXAMPLES = "examples"; + private static final String EXAMPLES_OUT = "examples-output"; + private static Configuration conf; + private static String HADOOP_OPTIONS; + + @BeforeClass + static void setUp() { + conf = new Configuration(); + conf.addResource('mapred-site.xml'); + HADOOP_OPTIONS = + "-fs ${conf.get('fs.default.name')} -jt ${conf.get('mapred.job.tracker')}"; + // Unpack resource + JarContent.unpackJarContainer(TestHadoopExamples.class, '.' , null) + + sh.exec("$hadoop fs $HADOOP_OPTIONS -test -e $EXAMPLES"); + if (sh.getRet() == 0) { + sh.exec("$hadoop fs $HADOOP_OPTIONS -rmr -skipTrash $EXAMPLES"); + assertTrue("Deletion of previous $EXAMPLES from HDFS failed", + sh.getRet() == 0); + } + sh.exec("$hadoop fs $HADOOP_OPTIONS -test -e $EXAMPLES_OUT"); + if (sh.getRet() == 0) { + sh.exec("$hadoop fs $HADOOP_OPTIONS -rmr -skipTrash $EXAMPLES_OUT"); + assertTrue("Deletion of previous examples output from HDFS failed", + sh.getRet() == 0); + } + +// copy test files to HDFS + sh.exec("hadoop fs $HADOOP_OPTIONS -put $EXAMPLES $EXAMPLES", + "hadoop fs $HADOOP_OPTIONS -mkdir $EXAMPLES_OUT"); + assertTrue("Could not create output directory", sh.getRet() == 0); + } + + static Map examples = + [ + pi :'20 10', + wordcount :"$EXAMPLES/text $EXAMPLES_OUT/wordcount", + multifilewc :"$EXAMPLES/text $EXAMPLES_OUT/multifilewc", +// aggregatewordcount:"$EXAMPLES/text $EXAMPLES_OUT/aggregatewordcount 5 textinputformat", +// aggregatewordhist :"$EXAMPLES/text $EXAMPLES_OUT/aggregatewordhist 5 textinputformat", + grep :"$EXAMPLES/text $EXAMPLES_OUT/grep '[Cc]uriouser'", + sleep :"-m 10 -r 10", + secondarysort :"$EXAMPLES/ints $EXAMPLES_OUT/secondarysort", + randomtextwriter :"-Dtest.randomtextwrite.total_bytes=1073741824 $EXAMPLES_OUT/randomtextwriter" + ]; + + private String testName; + private String testJar; + private String testArgs; + + @Parameters + public static Map<String, Object[]> generateTests() { + Map<String, Object[]> res = [:]; + examples.each { k, v -> res[k] = [k.toString(), v.toString()] as Object[]; } + return res; + } + + public TestHadoopExamples(String name, String args) { + testName = name; + testArgs = args; + testJar = HADOOP_EXAMPLES_JAR; + } + + @Test + void testMRExample() { + sh.exec("$hadoop jar $testJar $testName $HADOOP_OPTIONS $testArgs"); + + assertTrue("Example $testName failed", + sh.getRet() == 0); + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c8282c2/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy new file mode 100644 index 0000000..c7b0e18 --- /dev/null +++ b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoop/mapreduce/TestHadoopSmoke.groovy @@ -0,0 +1,95 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bigtop.itest.hadoop.mapreduce + +import org.apache.bigtop.itest.JarContent +import org.apache.bigtop.itest.shell.Shell +import org.junit.AfterClass +import org.junit.BeforeClass +import org.junit.Test +import org.apache.hadoop.conf.Configuration +import static org.junit.Assert.assertEquals + +// TODO: we have to stub it for 0.20.2 release, once we move to 0.21+ this can go +// import org.apache.hadoop.hdfs.DFSConfigKeys +class DFSConfigKeys { + static public final FS_DEFAULT_NAME_KEY = "fs.default.name"; +} + +class TestHadoopSmoke { + static Shell sh = new Shell("/bin/bash -s") + + static String hadoopHome = System.getProperty('HADOOP_HOME', '/thisfileneverwillexist') + static String testDir = "test.hadoopsmoke." + (new Date().getTime()) + static String nn = (new Configuration()).get(DFSConfigKeys.FS_DEFAULT_NAME_KEY) + + String cmd = "hadoop jar ${hadoopHome}/contrib/streaming/hadoop*streaming*.jar" + + " -D mapred.map.tasks=1 -D mapred.reduce.tasks=1 -D mapred.job.name=Experiment " + String cmd2 =" -input ${testDir}/cachefile/input.txt -mapper map.sh -file map.sh -reducer cat" + + " -output ${testDir}/cachefile/out -verbose " + String arg = "${nn}/user/${System.properties['user.name']}/${testDir}/cachefile/cachedir.jar#testlink " + + @BeforeClass + static void setUp() throws IOException { + JarContent.unpackJarContainer(TestHadoopSmoke.class, '.' , null) + + sh.exec( + "hadoop fs -mkdir ${testDir}/cachefile", + "hadoop dfs -put cachedir.jar ${testDir}/cachefile", + "hadoop dfs -put input.txt ${testDir}/cachefile", + ) + logError(sh) + } + + @AfterClass + static void tearDown() { + sh.exec("hadoop fs -rmr -skipTrash ${testDir}") + } + + @Test + void testCacheArchive() { + sh.exec("hadoop fs -rmr ${testDir}/cachefile/out", + cmd + ' -cacheArchive ' + arg + cmd2) + logError(sh) + sh.exec("hadoop fs -cat ${testDir}/cachefile/out/part-00000") + logError(sh) + + assertEquals("cache1\t\ncache2\t", sh.out.join('\n')) + } + + @Test + void testArchives() { + sh.exec("hadoop fs -rmr ${testDir}/cachefile/out", + cmd + ' -archives ' + arg + cmd2) + logError(sh) + sh.exec("hadoop fs -cat ${testDir}/cachefile/out/part-00000") + logError(sh) + + assertEquals("cache1\t\ncache2\t", sh.out.join('\n')) + } + + private static void logError (final Shell sh) { + if (sh.getRet()) { + println ('Failed command: ' + sh.script); + println ('\terror code: ' + sh.getRet()); + println ('\tstdout: ' + sh.getOut()); + println ('\tstderr: ' + sh.getErr()); + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c8282c2/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoopexamples/TestHadoopExamples.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoopexamples/TestHadoopExamples.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoopexamples/TestHadoopExamples.groovy deleted file mode 100644 index 9170e27..0000000 --- a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoopexamples/TestHadoopExamples.groovy +++ /dev/null @@ -1,125 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.bigtop.itest.hadoopexamples - -import org.junit.BeforeClass -import static org.junit.Assert.assertNotNull -import org.apache.bigtop.itest.shell.Shell -import static org.junit.Assert.assertTrue -import org.junit.Test -import org.apache.hadoop.conf.Configuration -import org.apache.bigtop.itest.JarContent -import org.apache.commons.logging.LogFactory -import org.apache.commons.logging.Log - -import org.apache.bigtop.itest.junit.OrderedParameterized -import org.junit.runners.Parameterized.Parameters -import org.junit.runner.RunWith - -@RunWith(OrderedParameterized.class) -class TestHadoopExamples { - static private Log LOG = LogFactory.getLog(TestHadoopExamples.class); - - private static final String HADOOP_HOME = System.getenv('HADOOP_HOME'); - private static final String HADOOP_CONF_DIR = System.getenv('HADOOP_CONF_DIR'); - private static String hadoopExamplesJar = - JarContent.getJarName(HADOOP_HOME, 'hadoop.*examples.*.jar'); - static { - assertNotNull("HADOOP_HOME has to be set to run this test", - HADOOP_HOME); - assertNotNull("HADOOP_CONF_DIR has to be set to run this test", - HADOOP_CONF_DIR); - assertNotNull("Can't find hadoop-examples.jar file", hadoopExamplesJar); - } - static final String HADOOP_EXAMPLES_JAR = - HADOOP_HOME + "/" + hadoopExamplesJar; - private static final String hadoop = "$HADOOP_HOME/bin/hadoop"; - - static Shell sh = new Shell("/bin/bash -s"); - private static final String EXAMPLES = "examples"; - private static final String EXAMPLES_OUT = "examples-output"; - private static Configuration conf; - private static String HADOOP_OPTIONS; - - @BeforeClass - static void setUp() { - conf = new Configuration(); - conf.addResource('mapred-site.xml'); - HADOOP_OPTIONS = - "-fs ${conf.get('fs.default.name')} -jt ${conf.get('mapred.job.tracker')}"; - // Unpack resource - JarContent.unpackJarContainer(TestHadoopExamples.class, '.' , null) - - sh.exec("$hadoop fs $HADOOP_OPTIONS -test -e $EXAMPLES"); - if (sh.getRet() == 0) { - sh.exec("$hadoop fs $HADOOP_OPTIONS -rmr -skipTrash $EXAMPLES"); - assertTrue("Deletion of previous $EXAMPLES from HDFS failed", - sh.getRet() == 0); - } - sh.exec("$hadoop fs $HADOOP_OPTIONS -test -e $EXAMPLES_OUT"); - if (sh.getRet() == 0) { - sh.exec("$hadoop fs $HADOOP_OPTIONS -rmr -skipTrash $EXAMPLES_OUT"); - assertTrue("Deletion of previous examples output from HDFS failed", - sh.getRet() == 0); - } - -// copy test files to HDFS - sh.exec("hadoop fs $HADOOP_OPTIONS -put $EXAMPLES $EXAMPLES", - "hadoop fs $HADOOP_OPTIONS -mkdir $EXAMPLES_OUT"); - assertTrue("Could not create output directory", sh.getRet() == 0); - } - - static Map examples = - [ - pi :'20 10', - wordcount :"$EXAMPLES/text $EXAMPLES_OUT/wordcount", - multifilewc :"$EXAMPLES/text $EXAMPLES_OUT/multifilewc", -// aggregatewordcount:"$EXAMPLES/text $EXAMPLES_OUT/aggregatewordcount 5 textinputformat", -// aggregatewordhist :"$EXAMPLES/text $EXAMPLES_OUT/aggregatewordhist 5 textinputformat", - grep :"$EXAMPLES/text $EXAMPLES_OUT/grep '[Cc]uriouser'", - sleep :"-m 10 -r 10", - secondarysort :"$EXAMPLES/ints $EXAMPLES_OUT/secondarysort", - randomtextwriter :"-Dtest.randomtextwrite.total_bytes=1073741824 $EXAMPLES_OUT/randomtextwriter" - ]; - - private String testName; - private String testJar; - private String testArgs; - - @Parameters - public static Map<String, Object[]> generateTests() { - Map<String, Object[]> res = [:]; - examples.each { k, v -> res[k] = [k.toString(), v.toString()] as Object[]; } - return res; - } - - public TestHadoopExamples(String name, String args) { - testName = name; - testArgs = args; - testJar = HADOOP_EXAMPLES_JAR; - } - - @Test - void testMRExample() { - sh.exec("$hadoop jar $testJar $testName $HADOOP_OPTIONS $testArgs"); - - assertTrue("Example $testName failed", - sh.getRet() == 0); - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c8282c2/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoopsmoke/TestHadoopSmoke.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoopsmoke/TestHadoopSmoke.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoopsmoke/TestHadoopSmoke.groovy deleted file mode 100644 index a4d1ab6..0000000 --- a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadoopsmoke/TestHadoopSmoke.groovy +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.bigtop.itest.hadoopsmoke - -import org.apache.bigtop.itest.JarContent -import org.apache.bigtop.itest.shell.Shell -import org.junit.AfterClass -import org.junit.BeforeClass -import org.junit.Test -import org.apache.hadoop.conf.Configuration -import static org.junit.Assert.assertEquals - -// TODO: we have to stub it for 0.20.2 release, once we move to 0.21+ this can go -// import org.apache.hadoop.hdfs.DFSConfigKeys -class DFSConfigKeys { - static public final FS_DEFAULT_NAME_KEY = "fs.default.name"; -} - -class TestHadoopSmoke { - static Shell sh = new Shell("/bin/bash -s") - - static String hadoopHome = System.getProperty('HADOOP_HOME', '/thisfileneverwillexist') - static String testDir = "test.hadoopsmoke." + (new Date().getTime()) - static String nn = (new Configuration()).get(DFSConfigKeys.FS_DEFAULT_NAME_KEY) - - String cmd = "hadoop jar ${hadoopHome}/contrib/streaming/hadoop*streaming*.jar" + - " -D mapred.map.tasks=1 -D mapred.reduce.tasks=1 -D mapred.job.name=Experiment " - String cmd2 =" -input ${testDir}/cachefile/input.txt -mapper map.sh -file map.sh -reducer cat" + - " -output ${testDir}/cachefile/out -verbose " - String arg = "${nn}/user/${System.properties['user.name']}/${testDir}/cachefile/cachedir.jar#testlink " - - @BeforeClass - static void setUp() throws IOException { - JarContent.unpackJarContainer(TestHadoopSmoke.class, '.' , null) - - sh.exec( - "hadoop fs -mkdir ${testDir}/cachefile", - "hadoop dfs -put cachedir.jar ${testDir}/cachefile", - "hadoop dfs -put input.txt ${testDir}/cachefile", - ) - logError(sh) - } - - @AfterClass - static void tearDown() { - sh.exec("hadoop fs -rmr -skipTrash ${testDir}") - } - - @Test - void testCacheArchive() { - sh.exec("hadoop fs -rmr ${testDir}/cachefile/out", - cmd + ' -cacheArchive ' + arg + cmd2) - logError(sh) - sh.exec("hadoop fs -cat ${testDir}/cachefile/out/part-00000") - logError(sh) - - assertEquals("cache1\t\ncache2\t", sh.out.join('\n')) - } - - @Test - void testArchives() { - sh.exec("hadoop fs -rmr ${testDir}/cachefile/out", - cmd + ' -archives ' + arg + cmd2) - logError(sh) - sh.exec("hadoop fs -cat ${testDir}/cachefile/out/part-00000") - logError(sh) - - assertEquals("cache1\t\ncache2\t", sh.out.join('\n')) - } - - private static void logError (final Shell sh) { - if (sh.getRet()) { - println ('Failed command: ' + sh.script); - println ('\terror code: ' + sh.getRet()); - println ('\tstdout: ' + sh.getOut()); - println ('\tstderr: ' + sh.getErr()); - } - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c8282c2/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadooptests/TestTestCLI.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadooptests/TestTestCLI.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadooptests/TestTestCLI.groovy deleted file mode 100644 index 3a6211c..0000000 --- a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadooptests/TestTestCLI.groovy +++ /dev/null @@ -1,57 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.bigtop.itest.hadooptests - -import org.junit.Ignore -import org.junit.Test -import org.apache.hadoop.conf.Configuration -import org.apache.hadoop.cli.* - -// TODO: we have to stub it for 0.20.2 release, once we move to 0.21+ this can go -// import org.apache.hadoop.fs.CommonConfigurationKeys -class CommonConfigurationKeys { - static final public String FS_DEFAULT_NAME_KEY = "fs.default.name"; -} - -/** This test class serves only one purpose: to prepare environment for proper - execution of parent (TestCLI) - */ -class TestTestCLI extends TestCLI { - private static final String JT = "mapred.job.tracker"; - private static Configuration conf = new Configuration(); - - static { - conf.addDefaultResource('mapred-site.xml'); - System.properties.setProperty(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY, - conf.get(CommonConfigurationKeys.FS_DEFAULT_NAME_KEY)); - System.properties.setProperty(JT, conf.get(JT)); - System.properties.setProperty('test.cli.config', 'testConfCluster.xml'); - // The following doesn't really work because in the parent this property - // is used to set a value to a static member. Basically, by the time - // we get here it's too late already. - // The property needs to be set from runtime - System.properties.setProperty('test.cache.data', 'clitest_data'); - } - - @Test - @Ignore("HADOOP-7730") - @Override - public void testAll() { - super.testAll(); - } -} http://git-wip-us.apache.org/repos/asf/bigtop/blob/0c8282c2/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadooptests/TestTextSnappy.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadooptests/TestTextSnappy.groovy b/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadooptests/TestTextSnappy.groovy deleted file mode 100644 index 6142979..0000000 --- a/bigtop-tests/test-artifacts/hadoop/src/main/groovy/org/apache/bigtop/itest/hadooptests/TestTextSnappy.groovy +++ /dev/null @@ -1,67 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p/> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p/> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.bigtop.itest.hadooptests - -import org.apache.bigtop.itest.shell.Shell -import org.junit.AfterClass -import org.junit.BeforeClass -import org.junit.Test -import static org.junit.Assert.assertEquals - -class TestTextSnappy { - static Shell sh = new Shell("/bin/bash -s") - static String testDir = "testtextsnappy." + (new Date().getTime()) - static String snappyFile = "part-00001.snappy" - - @BeforeClass - static void setUp() throws IOException { - sh.exec( - "hadoop fs -mkdir ${testDir}", - "hadoop fs -put ${snappyFile} ${testDir}/${snappyFile}", - ) - logError(sh) - } - - @AfterClass - static void tearDown() { - sh.exec("hadoop fs -rmr -skipTrash ${testDir}") - } - - @Test - void testTextSnappy() { - String cmd = "hadoop fs -text ${testDir}/${snappyFile}" - System.out.println(cmd) - sh.exec(cmd) - String output = sh.getOut().join("\n") - logError(sh) - String expected = "1\trafferty\t31\n2\tjones\t33\n3\tsteinberg\t33" - System.out.println("Expected output:\n${expected}") - System.out.println("Actual output:\n${output}") - assertEquals("Incorrect output", output, expected) - } - - private static void logError (final Shell sh) { - if (sh.getRet()) { - println ('Failed command: ' + sh.script); - println ('\terror code: ' + sh.getRet()); - println ('\tstdout: ' + sh.getOut()); - println ('\tstderr: ' + sh.getErr()); - } - } -}
