Repository: bigtop Updated Branches: refs/heads/master f6eac001a -> 6f2a5fc9f
1821: added ignite smoke tests ported from grid gain tests. Signed-off-by: Konstantin Boudnik <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/6f2a5fc9 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/6f2a5fc9 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/6f2a5fc9 Branch: refs/heads/master Commit: 6f2a5fc9f6a61dc1163acbfe9387d784f96e313b Parents: f6eac00 Author: iveselovskiy <[email protected]> Authored: Tue Aug 4 15:12:22 2015 +0300 Committer: Konstantin Boudnik <[email protected]> Committed: Fri Aug 7 14:33:39 2015 -0700 ---------------------------------------------------------------------- .../ignite-hadoop/TestIgniteHadoop.groovy | 72 ++++++++++++++++ .../smoke-tests/ignite-hadoop/build.gradle | 53 ++++++++++++ .../ignite-hadoop/conf/core-site.xml | 90 ++++++++++++++++++++ .../ignite-hadoop/conf/log4j.properties | 24 ++++++ .../ignite-hadoop/conf/mapred-site.xml | 66 ++++++++++++++ .../smoke-tests/ignite-hadoop/test.data | 11 +++ pom.xml | 1 + 7 files changed, 317 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/TestIgniteHadoop.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/TestIgniteHadoop.groovy b/bigtop-tests/smoke-tests/ignite-hadoop/TestIgniteHadoop.groovy new file mode 100644 index 0000000..6358b7f --- /dev/null +++ b/bigtop-tests/smoke-tests/ignite-hadoop/TestIgniteHadoop.groovy @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.bigtop.itest.hadoop.ignite + +import org.apache.bigtop.itest.shell.Shell +import org.apache.commons.logging.Log +import org.apache.commons.logging.LogFactory +import org.junit.Before +import org.junit.Test + +import static org.junit.Assert.assertEquals + +class TestIgniteHadoop { + static private Log LOG = LogFactory.getLog(Object.class) + + static Shell sh = new Shell("/bin/bash -s") + + static String hadoopClassPath = "/usr/lib/ignite-hadoop/libs/*:/usr/lib/ignite-hadoop/libs/ignite-hadoop/*" + + static String cmdPrefix = "export HADOOP_CLASSPATH=$hadoopClassPath:\$HADOOP_CLASSPATH; " + + "export HADOOP_CONF_DIR=\$(pwd)/conf;" + + + private static void execCommand(String cmd) { + LOG.info(cmd) + + sh.exec("$cmdPrefix $cmd") + } + + @Before + void cleanFileSystem() { + execCommand("hadoop fs -rm -r /gh-input") + execCommand("hadoop fs -rm -r /gh-output") + } + + @Test + void test() { + execCommand("hadoop fs -mkdir /gh-input") + execCommand("hadoop fs -put test.data /gh-input/") + + execCommand("hadoop jar \$HADOOP_MAPRED_HOME/hadoop-mapreduce-examples.jar wordcount /gh-input /gh-output") + + execCommand("hadoop fs -cat /gh-output/part-r-00000") + + String expected = + "black\t5\n" + + "blue\t11\n" + + "green\t11\n" + + "white\t5\n" + + "yellow\t11"; + + String actual = sh.out.join('\n') + + assertEquals("Incorrect output", expected, actual) + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/build.gradle ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/build.gradle b/bigtop-tests/smoke-tests/ignite-hadoop/build.gradle new file mode 100644 index 0000000..f0973ff --- /dev/null +++ b/bigtop-tests/smoke-tests/ignite-hadoop/build.gradle @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +apply plugin: 'groovy' + +repositories { + mavenCentral() +} + +dependencies { + //needed to avoid groovy not on classpath error. + testCompile module('org.codehaus.groovy:groovy:1.8.0') + testCompile group: 'org.apache.bigtop.itest', name: 'itest-common', version: itestVersion, transitive: 'true' + testCompile group: 'org.apache.hadoop', name: 'hadoop-common', version: hadoopVersion, transitive: 'true' + testCompile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: hadoopVersion, transitive: 'true' +} + +def tests_to_include() { + return [ + "TestIgniteHadoop.groovy" + ]; +} + +sourceSets { + test { + resources { + srcDirs = [ + 'conf/' + ] + } + groovy { + srcDirs = ["$System.env.BIGTOP_HOME"+"/bigtop-tests/smoke-tests/ignite-hadoop/"] + } + } +} + +test.doFirst { + checkEnv(["BIGTOP_HOME", "HADOOP_MAPRED_HOME"]) +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/conf/core-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/conf/core-site.xml b/bigtop-tests/smoke-tests/ignite-hadoop/conf/core-site.xml new file mode 100644 index 0000000..8b8e634 --- /dev/null +++ b/bigtop-tests/smoke-tests/ignite-hadoop/conf/core-site.xml @@ -0,0 +1,90 @@ +<?xml version="1.0" encoding="UTF-8"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + This template file contains settings needed to run Apache Hadoop jobs + with Apache Ignite's distributed in-memory file system IGFS. + + You can replace '$HADOOP_HOME/etc/hadoop/core-site.xml' file with this one + to work with IGFS nodes running on localhost (these local nodes can be + a part of distributed cluster though). To work with file system on remote + hosts you need to change the host of file system URI to any host running + IGFS node. + + Note that Ignite jars must be in Apache Hadoop client classpath to work + with this configuration. + + Run script '$IGNITE_HOME/bin/setup-hadoop.{sh|bat}' for Apache Hadoop client setup. +--> + +<configuration> + <!-- + Set default file system to IGFS instance named "igfs" configured in Ignite. + --> + <property> + <name>fs.default.name</name> + <value>igfs://igfs@localhost</value> + </property> + + <!-- + Set Hadoop 1.* file system implementation class for IGFS. + --> + <property> + <name>fs.igfs.impl</name> + <value>org.apache.ignite.hadoop.fs.v1.IgniteHadoopFileSystem</value> + </property> + + <!-- + Set Hadoop 2.* file system implementation class for IGFS. + --> + <property> + <name>fs.AbstractFileSystem.igfs.impl</name> + <value>org.apache.ignite.hadoop.fs.v2.IgniteHadoopFileSystem</value> + </property> + + <!-- + Disallow data node replacement since it does not make sense for IGFS nodes. + --> + <property> + <name>dfs.client.block.write.replace-datanode-on-failure.policy</name> + <value>NEVER</value> + </property> + + <!-- + Allow to write the job statistics into IGFS. + --> + <!-- + <property> + <name>ignite.counters.writer</name> + <value>org.apache.ignite.hadoop.fs.IgniteHadoopFileSystemCounterWriter</value> + </property> + --> + + <!-- + By default data is placed into the file /user/<user_name>/<job_id>/performance + You can override this path with using macro ${USER} that is to injection of submitter user name. + --> + <!-- + <property> + <name>ignite.counters.fswriter.directory</name> + <value>/user/${USER}</value> + </property> + --> +</configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/conf/log4j.properties ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/conf/log4j.properties b/bigtop-tests/smoke-tests/ignite-hadoop/conf/log4j.properties new file mode 100644 index 0000000..0723326 --- /dev/null +++ b/bigtop-tests/smoke-tests/ignite-hadoop/conf/log4j.properties @@ -0,0 +1,24 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Set root logger level to INFO and its only appender to A1. +log4j.rootLogger=INFO, A1 + +# A1 is set to be a ConsoleAppender. +log4j.appender.A1=org.apache.log4j.ConsoleAppender + +# A1 uses PatternLayout. +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%-4r [BIGTOP-TEST-LOG %t] %-5p %c %x --- %m%n http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/conf/mapred-site.xml ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/conf/mapred-site.xml b/bigtop-tests/smoke-tests/ignite-hadoop/conf/mapred-site.xml new file mode 100644 index 0000000..a2ed437 --- /dev/null +++ b/bigtop-tests/smoke-tests/ignite-hadoop/conf/mapred-site.xml @@ -0,0 +1,66 @@ +<?xml version="1.0"?> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> + +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<!-- + This template file contains settings needed to run Apache Hadoop jobs + with Apache Ignite In-Memory Accelerator. + + You can replace '$HADOOP_HOME/etc/hadoop/mapred-site.xml' file with this one + to run jobs on localhost (local node can be a part of distributed cluster though). + To run jobs on remote host you have to change jobtracker address to the REST address + of any running Ignite node. + + Note that Ignite jars must be in Apache Hadoop client classpath to work + with this configuration. + + Run script '$IGNITE_HOME/bin/setup-hadoop.{sh|bat}' for Apache Hadoop client setup. +--> + +<configuration> + <!-- + Framework name must be set to 'ignite'. + --> + <property> + <name>mapreduce.framework.name</name> + <value>ignite</value> + </property> + + <!-- + Job tracker address must be set to the REST address of any running Ignite node. + --> + <property> + <name>mapreduce.jobtracker.address</name> + <value>localhost:11211</value> + </property> + + <!-- Parameters for job tuning. --> + <!-- + <property> + <name>mapreduce.job.reduces</name> + <value>1</value> + </property> + + <property> + <name>mapreduce.job.maps</name> + <value>4</value> + </property> + --> + +</configuration> http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/bigtop-tests/smoke-tests/ignite-hadoop/test.data ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/ignite-hadoop/test.data b/bigtop-tests/smoke-tests/ignite-hadoop/test.data new file mode 100644 index 0000000..6599669 --- /dev/null +++ b/bigtop-tests/smoke-tests/ignite-hadoop/test.data @@ -0,0 +1,11 @@ +green blue yellow +yellow green blue black white +yellow green blue +white yellow green blue black +blue yellow green +black white yellow green blue +green blue yellow +blue black white yellow green +yellow green blue +green blue black white yellow +blue yellow green http://git-wip-us.apache.org/repos/asf/bigtop/blob/6f2a5fc9/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 455f35a..49e9e1d 100644 --- a/pom.xml +++ b/pom.xml @@ -304,6 +304,7 @@ <exclude>bigtop-tests/smoke-tests/kite/sandwiches.csv</exclude> <exclude>bigtop-tests/smoke-tests/phoenix/*.csv</exclude> <exclude>bigtop-tests/smoke-tests/phoenix/*.sql</exclude> + <exclude>bigtop-tests/smoke-tests/ignite-hadoop/*.data</exclude> <exclude>**/target/**</exclude> <exclude>**/build/**</exclude> <exclude>**/.gradle/**</exclude>
