Added tests into TestSpecsRuntime and testRuntimeSpecConf so they get run. Tested against an HDP cluster.
(cherry picked from commit a911703ccacc70b738720c560574fc8a9aee8b2b) Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/77e0d6e0 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/77e0d6e0 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/77e0d6e0 Branch: refs/heads/master Commit: 77e0d6e01f3e43f4a7e51a588b2261780c587ae2 Parents: c65c000 Author: Alan Gates <[email protected]> Authored: Thu Mar 9 15:39:13 2017 -0800 Committer: Roman Shaposhnik <[email protected]> Committed: Thu Mar 23 10:27:15 2017 -0700 ---------------------------------------------------------------------- .../odpi/specs/runtime/hadoop/ApiExaminer.java | 19 +----- .../src/main/resources/api-examiner-checker.sh | 52 --------------- .../src/main/resources/api-examiner-prep.sh | 38 ++++++++--- .../odpi/specs/runtime/TestSpecsRuntime.groovy | 32 ++++++++++ .../test/resources/testRuntimeSpecConf.groovy | 66 ++++++++++++++++++++ 5 files changed, 130 insertions(+), 77 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/77e0d6e0/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java b/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java index 2ae97a2..d95c010 100644 --- a/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java +++ b/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java @@ -1,11 +1,7 @@ /** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at * <p> * http://www.apache.org/licenses/LICENSE-2.0 * <p> @@ -62,15 +58,6 @@ public class ApiExaminer { unloadableClasses.add("org.apache.hadoop.io.compress.lz4.Lz4Compressor"); unloadableClasses.add("org.apache.hadoop.record.compiler.ant.RccTask"); - /* - jarsWeCareAbout = new ArrayList<>(); - jarsWeCareAbout.add("hadoop-common"); - jarsWeCareAbout.add("hadoop-hdfs"); - jarsWeCareAbout.add("hadoop-yarn-common"); - jarsWeCareAbout.add("hadoop-yarn-client"); - jarsWeCareAbout.add("hadoop-yarn-api"); - jarsWeCareAbout.add("hadoop-mapreduce-client-core"); - */ } public static void main(String[] args) { http://git-wip-us.apache.org/repos/asf/bigtop/blob/77e0d6e0/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-checker.sh ---------------------------------------------------------------------- diff --git a/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-checker.sh b/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-checker.sh deleted file mode 100755 index 1cb9583..0000000 --- a/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-checker.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env bash - -############################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# <p> -# http://www.apache.org/licenses/LICENSE-2.0 -# <p> -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -############################################################################ - -function usage() { - echo "You must set the following variables: HADOOP_COMMON_HOME HADOOP_COMMON_DIR HADOOP_COMMON_LIB_JARS_DIR " - echo "HADOOP_HDFS_HOME HDFS_DIR HDFS_LIB_JARS_DIR HADOOP_YARN_HOME YARN_DIR YARN_LIB_JARS_DIR " - echo "HADOOP_MAPRED_HOME MAPRED_DIR MAPRED_LIB_JARS_DIR BIGTOP_HOME (location of bigtop source)" - echo "You can get the Hadoop environment variables by using hadoop envvars, hdfs envvars, yarn envvars, and mapred envvars" -} - -for envar in x$HADOOP_COMMON_HOME x$HADOOP_COMMON_DIR x$HADOOP_COMMON_LIB_JARS_DIR x$HADOOP_HDFS_HOME x$HDFS_DIR \ - x$HDFS_LIB_JARS_DIR x$HADOOP_YARN_HOME x$YARN_DIR x$YARN_LIB_JARS_DIR x$HADOOP_MAPRED_HOME x$MAPRED_DIR \ - x$MAPRED_LIB_JARS_DIR -do - if [ "${envar}" = "x" ] - then - usage - exit 1 - fi -done - - -for dir in $BIGTOP_HOME/bigtop-tests/spec-tests/runtime/build/libs/ $HADOOP_COMMON_HOME/$HADOOP_COMMON_DIR \ - $HADOOP_COMMON_HOME/$HADOOP_COMMON_LIB_JARS_DIR $HADOOP_HDFS_HOME/$HDFS_DIR \ - $HADOOP_HDFS_HOME/$HDFS_LIB_JARS_DIR $HADOOP_YARN_HOME/$YARN_DIR $HADOOP_YARN_HOME/$YARN_LIB_JARS_DIR \ - $HADOOP_MAPRED_HOME/$MAPRED_DIR $HADOOP_MAPRED_HOME/$MAPRED_LIB_JARS_DIR -do - for jar in `find $dir -name \*.jar` - do - CLASSPATH=$CLASSPATH:$jar - done -done - -java -cp $CLASSPATH org.odpi.specs.runtime.hadoop.ApiExaminer $@ - http://git-wip-us.apache.org/repos/asf/bigtop/blob/77e0d6e0/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-prep.sh ---------------------------------------------------------------------- diff --git a/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-prep.sh b/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-prep.sh index 0bdfe05..8c9ab5e 100755 --- a/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-prep.sh +++ b/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner-prep.sh @@ -1,16 +1,12 @@ #!/usr/bin/env bash ############################################################################ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# <p> +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# # http://www.apache.org/licenses/LICENSE-2.0 -# <p> +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -18,6 +14,30 @@ # limitations under the License. ############################################################################ +############################################################################ +# This script is used to generate the hadoop-*-api.report.json files in the +# test/resources directory. To use it, you will first need to download an +# Apache binary distribution of Hadoop and set APACHE_HADOOP_DIR to the +# directory where you untar that distribution. You will then need to set +# BIGTTOP_HOME to the directory where your bigtop source is located. Then +# run this script for each of the jars you want to generate a report for. +# The arguments passed to this script should be -p <outputdir> -j <jarfile> +# where outputdir is the directory you'd like to write the report to and +# jarfile is the full path of the jar to generate the report for. Reports +# should be generated for the following jars: hadoop-common, hadoop-hdfs, +# hadoop-yarn-common, hadoop-yarn-client, hadoop-yarn-api, and +# hadoop-mapreduce-client-core +# +# Example usage: +# export APACHE_HADOOP_DIR=/tmp/hadoop-2.7.3 +# export BIGTOP_HOME=/home/me/git/bigtop +# $BIGTOP_HOME/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner.sh -j $HADOOP_HOME/share/hadoop/common/hadoop-common-2.7.3.jar -p $BIGTOP_HOME/bigtop-tests/spec-tests/runtime/src/test/resources +# +# The resulting reports should be committed to git. This script only needs +# to be run once per ODPi release. +############################################################################ + + if [ "x${APACHE_HADOOP_DIR}" = "x" ] then echo "You must set APACHE_HADOOP_DIR to the directory you have placed the Apache Hadoop binary distribution in" http://git-wip-us.apache.org/repos/asf/bigtop/blob/77e0d6e0/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy b/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy index 2426278..bc2a3b2 100644 --- a/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy +++ b/bigtop-tests/spec-tests/runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy @@ -25,6 +25,9 @@ import org.junit.runner.RunWith import org.junit.runners.Parameterized import org.junit.runners.Parameterized.Parameters +import java.util.regex.Matcher +import java.util.regex.Pattern + /** * Check all expected environment * Tests are constructed dynamically, using external DSL to define @@ -236,6 +239,35 @@ public class TestSpecsRuntime { } ) break + case 'api_examination': + def basedir = getEnv(arguments['baseDirEnv'], arguments['envcmd']) + def libdir = getEnv(arguments['libDir'], arguments['envcmd']) + + def dir = new File(basedir + "/" + libdir) + Assert.assertTrue("Expected " + dir.getPath() + " to be a directory", dir.isDirectory()) + def pattern = Pattern.compile(arguments['jar'] + "-[0-9]+.*\\.jar") + def String[] jars = dir.list(new FilenameFilter() { + @Override + boolean accept(File d, String name) { + Matcher matcher = pattern.matcher(name) + return (matcher.matches() && !name.contains("test")) + } + }) + Assert.assertEquals("Expected only one jar, but got " + jars.join(", "), 1, jars.length) + def jar = dir.getAbsolutePath() + "/" + jars[0] + + def examinerJar = System.properties['odpi.test.hive.hcat.job.jar'] + def resourceFile = System.properties['test.resources.dir']+ "/" + arguments['resourceFile'] + Shell sh = new Shell() + def results = sh.exec("hadoop jar " + examinerJar + " org.odpi.specs.runtime.hadoop.ApiExaminer -c " + resourceFile + " -j " + jar).getErr() + int rc = sh.getRet() + Assert.assertEquals("Expected command to succeed, but got return code " + rc, 0, rc) + if (results.size() > 0) { + System.out.println("Received report for jar " + arguments['jar'] + results.join("\n")) + } + break; + + default: break } http://git-wip-us.apache.org/repos/asf/bigtop/blob/77e0d6e0/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy b/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy index f5521c5..339de4c 100644 --- a/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy +++ b/bigtop-tests/spec-tests/runtime/src/test/resources/testRuntimeSpecConf.groovy @@ -360,5 +360,71 @@ specs { arguments { } } + 'HADOOP_API1' { + name = "HADOOP_API1" + type = 'api_examination' + arguments { + baseDirEnv = 'HADOOP_COMMON_HOME' + libDir = 'HADOOP_COMMON_DIR' + envcmd = 'hadoop envvars' + jar = 'hadoop-common' + resourceFile = 'hadoop-common-2.7.3-api-report.json' + } + } + 'HADOOP_API2' { + name = "HADOOP_API2" + type = 'api_examination' + arguments { + baseDirEnv = 'HADOOP_HDFS_HOME' + libDir = 'HDFS_DIR' + envcmd = 'hdfs envvars' + jar = 'hadoop-hdfs' + resourceFile = 'hadoop-hdfs-2.7.3-api-report.json' + } + } + 'HADOOP_API3' { + name = "HADOOP_API3" + type = 'api_examination' + arguments { + baseDirEnv = 'HADOOP_YARN_HOME' + libDir = 'YARN_DIR' + envcmd = 'yarn envvars' + jar = 'hadoop-yarn-common' + resourceFile = 'hadoop-yarn-common-2.7.3-api-report.json' + } + } + 'HADOOP_API4' { + name = "HADOOP_API4" + type = 'api_examination' + arguments { + baseDirEnv = 'HADOOP_YARN_HOME' + libDir = 'YARN_DIR' + envcmd = 'yarn envvars' + jar = 'hadoop-yarn-client' + resourceFile = 'hadoop-yarn-client-2.7.3-api-report.json' + } + } + 'HADOOP_API5' { + name = "HADOOP_API5" + type = 'api_examination' + arguments { + baseDirEnv = 'HADOOP_YARN_HOME' + libDir = 'YARN_DIR' + envcmd = 'yarn envvars' + jar = 'hadoop-yarn-api' + resourceFile = 'hadoop-yarn-api-2.7.3-api-report.json' + } + } + 'HADOOP_API6' { + name = "HADOOP_API6" + type = 'api_examination' + arguments { + baseDirEnv = 'HADOOP_MAPRED_HOME' + libDir = 'MAPRED_DIR' + envcmd = 'mapred envvars' + jar = 'hadoop-mapreduce-client-core' + resourceFile = 'hadoop-mapreduce-client-core-2.7.3-api-report.json' + } + } } }
