BIGTOP-2704. Include ODPi runtime tests option into the battery of smoke tests
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/5e342c45 Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/5e342c45 Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/5e342c45 Branch: refs/heads/master Commit: 5e342c45364ec97f5e3530769a1cc8bdbcf69bb0 Parents: 77e0d6e Author: Roman Shaposhnik <[email protected]> Authored: Wed Mar 22 08:51:22 2017 -0700 Committer: Roman Shaposhnik <[email protected]> Committed: Thu Mar 23 10:27:16 2017 -0700 ---------------------------------------------------------------------- bigtop-tests/smoke-tests/odpi-runtime/README.md | 48 ++ .../smoke-tests/odpi-runtime/build.gradle | 63 +++ .../odpi/specs/runtime/hadoop/ApiExaminer.java | 485 +++++++++++++++++ .../org/odpi/specs/runtime/hive/HCatalogMR.java | 137 +++++ .../src/main/resources/api-examiner-prep.sh | 64 +++ .../odpi/specs/runtime/TestSpecsRuntime.groovy | 275 ++++++++++ .../org/odpi/specs/runtime/hive/HiveHelper.java | 121 ++++ .../odpi/specs/runtime/hive/JdbcConnector.java | 79 +++ .../odpi/specs/runtime/hive/TestBeeline.java | 201 +++++++ .../org/odpi/specs/runtime/hive/TestCLI.java | 213 ++++++++ .../odpi/specs/runtime/hive/TestHCatalog.java | 158 ++++++ .../org/odpi/specs/runtime/hive/TestJdbc.java | 545 +++++++++++++++++++ .../org/odpi/specs/runtime/hive/TestSql.java | 337 ++++++++++++ .../org/odpi/specs/runtime/hive/TestThrift.java | 251 +++++++++ .../src/test/python/find-public-apis.py | 80 +++ .../hadoop-common-2.7.3-api-report.json | 1 + .../src/test/resources/hadoop-common-bin.list | 2 + .../src/test/resources/hadoop-common-jar.list | 60 ++ .../src/test/resources/hadoop-common.list | 230 ++++++++ .../resources/hadoop-hdfs-2.7.3-api-report.json | 1 + .../src/test/resources/hadoop-hdfs-bin.list | 1 + .../src/test/resources/hadoop-hdfs-jar.list | 25 + .../src/test/resources/hadoop-hdfs.list | 79 +++ .../test/resources/hadoop-mapreduce-bin.list | 1 + ...-mapreduce-client-core-2.7.3-api-report.json | 1 + .../test/resources/hadoop-mapreduce-jar.list | 22 + .../src/test/resources/hadoop-mapreduce.list | 123 +++++ .../src/test/resources/hadoop-subprojs.list | 4 + .../hadoop-yarn-api-2.7.3-api-report.json | 1 + .../src/test/resources/hadoop-yarn-bin.list | 3 + .../hadoop-yarn-client-2.7.3-api-report.json | 1 + .../hadoop-yarn-common-2.7.3-api-report.json | 1 + .../src/test/resources/hadoop-yarn-jar.list | 38 ++ .../src/test/resources/hadoop-yarn.list | 74 +++ .../test/resources/testRuntimeSpecConf.groovy | 430 +++++++++++++++ bigtop-tests/spec-tests/README.md | 48 -- bigtop-tests/spec-tests/build.gradle | 63 --- bigtop-tests/spec-tests/runtime/build.gradle | 63 --- .../odpi/specs/runtime/hadoop/ApiExaminer.java | 485 ----------------- .../org/odpi/specs/runtime/hive/HCatalogMR.java | 137 ----- .../src/main/resources/api-examiner-prep.sh | 64 --- .../odpi/specs/runtime/TestSpecsRuntime.groovy | 275 ---------- .../org/odpi/specs/runtime/hive/HiveHelper.java | 121 ---- .../odpi/specs/runtime/hive/JdbcConnector.java | 79 --- .../odpi/specs/runtime/hive/TestBeeline.java | 201 ------- .../org/odpi/specs/runtime/hive/TestCLI.java | 213 -------- .../odpi/specs/runtime/hive/TestHCatalog.java | 158 ------ .../org/odpi/specs/runtime/hive/TestJdbc.java | 545 ------------------- .../org/odpi/specs/runtime/hive/TestSql.java | 337 ------------ .../org/odpi/specs/runtime/hive/TestThrift.java | 251 --------- .../runtime/src/test/python/find-public-apis.py | 80 --- .../hadoop-common-2.7.3-api-report.json | 1 - .../src/test/resources/hadoop-common-bin.list | 2 - .../src/test/resources/hadoop-common-jar.list | 60 -- .../src/test/resources/hadoop-common.list | 230 -------- .../resources/hadoop-hdfs-2.7.3-api-report.json | 1 - .../src/test/resources/hadoop-hdfs-bin.list | 1 - .../src/test/resources/hadoop-hdfs-jar.list | 25 - .../runtime/src/test/resources/hadoop-hdfs.list | 79 --- .../test/resources/hadoop-mapreduce-bin.list | 1 - ...-mapreduce-client-core-2.7.3-api-report.json | 1 - .../test/resources/hadoop-mapreduce-jar.list | 22 - .../src/test/resources/hadoop-mapreduce.list | 123 ----- .../src/test/resources/hadoop-subprojs.list | 4 - .../hadoop-yarn-api-2.7.3-api-report.json | 1 - .../src/test/resources/hadoop-yarn-bin.list | 3 - .../hadoop-yarn-client-2.7.3-api-report.json | 1 - .../hadoop-yarn-common-2.7.3-api-report.json | 1 - .../src/test/resources/hadoop-yarn-jar.list | 38 -- .../runtime/src/test/resources/hadoop-yarn.list | 74 --- .../test/resources/testRuntimeSpecConf.groovy | 430 --------------- build.gradle | 3 +- settings.gradle | 5 - 73 files changed, 4157 insertions(+), 4224 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/5e342c45/bigtop-tests/smoke-tests/odpi-runtime/README.md ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/odpi-runtime/README.md b/bigtop-tests/smoke-tests/odpi-runtime/README.md new file mode 100644 index 0000000..8fde997 --- /dev/null +++ b/bigtop-tests/smoke-tests/odpi-runtime/README.md @@ -0,0 +1,48 @@ +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +Test suite to validate Hadoop basic specifications +================================================== + +The test suite is intended to be used as a validation tool to make sure that a +Hadoop stack derived from Apache Bigtop is still compliant with it. The +minimalistic way of doing so would be to guarantee compatibility of the +environment, binaries layouts, certain configuration parameters, and so on. + +Validation test suite for the specs is vaguely based on Apache Bigtop iTest and +consists of two essential parts: a configuration file, communicating the +functional commands and expected outcome(s) of it; and the test driver to run +the commands and compare the results. + +Running the tests +================= + +Tests could be executed by running the following command +``` + gradle :bigtop-tests:spec-tests:runtime:test -Pspec.tests --info +``` +======= +consists of two essential parts: a configuration file, communicating the +functional commands and expected outcome(s) of it; and the test driver to run +the commands and compare the results. + +Running the tests +================= + +Tests could be executed by running the following command +``` + gradle :bigtop-tests:spec-tests:runtime:test -Pspec.tests --info +``` + http://git-wip-us.apache.org/repos/asf/bigtop/blob/5e342c45/bigtop-tests/smoke-tests/odpi-runtime/build.gradle ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/odpi-runtime/build.gradle b/bigtop-tests/smoke-tests/odpi-runtime/build.gradle new file mode 100644 index 0000000..97e3635 --- /dev/null +++ b/bigtop-tests/smoke-tests/odpi-runtime/build.gradle @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +def junitVersion = '4.11' + +apply plugin: 'java' + +repositories { + maven { + url "http://conjars.org/repo/"; + } +} +dependencies { + compile group: 'junit', name: 'junit', version: junitVersion, transitive: 'true' + compile group: 'commons-logging', name: 'commons-logging', version: '1.1.3' + compile group: 'org.apache.commons', name: 'commons-exec', version: '1.3' + compile group: 'org.apache.hive', name: 'hive-jdbc', version: '1.2.1' + compile group: 'org.apache.hive', name: 'hive-metastore', version: '1.2.1' + compile group: 'org.apache.hive', name: 'hive-common', version: '1.2.1' + compile group: 'org.apache.thrift', name: 'libfb303', version: '0.9.3' + compile group: 'org.apache.thrift', name: 'libthrift', version: '0.9.3' + compile group: 'org.apache.hadoop', name: 'hadoop-common', version: '2.7.2' + compile group: 'org.apache.hive.hcatalog', name: 'hive-hcatalog-core', version: '1.2.1' + testCompile group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-core', version: '2.7.2' + compile group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-jobclient', version: '2.7.2' + testCompile group: 'org.apache.hadoop', name: 'hadoop-mapreduce-client-common', version: '2.7.2' + testCompile group: 'org.apache.hadoop', name: 'hadoop-hdfs', version: '2.7.2' + testCompile group: 'org.apache.hive', name: 'hive-exec', version: '1.2.1' + testCompile "junit:junit:4.11" + if (System.env.HADOOP_CONF_DIR) testRuntime files(System.env.HADOOP_CONF_DIR) +} + +jar { + from { + (configurations.runtime).grep{it.toString() =~ /(hive|libfb303)-.*[jw]ar$/}.collect { + zipTree(it) + } + } + + exclude 'META-INF/*.RSA', 'META-INF/*.SF','META-INF/*.DSA' +} + +test { + // Change the default location where test data is picked up + systemProperty 'test.resources.dir', "${buildDir}/resources/test/" + systemProperty 'odpi.test.hive.hcat.job.jar', jar.archivePath + systemProperty 'odpi.test.hive.hcat.core.jar', (configurations.runtime).find { it.toString() =~ /hive-hcatalog-core-.*jar$/ } +} +test.dependsOn jar http://git-wip-us.apache.org/repos/asf/bigtop/blob/5e342c45/bigtop-tests/smoke-tests/odpi-runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/odpi-runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java b/bigtop-tests/smoke-tests/odpi-runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java new file mode 100644 index 0000000..d95c010 --- /dev/null +++ b/bigtop-tests/smoke-tests/odpi-runtime/src/main/java/org/odpi/specs/runtime/hadoop/ApiExaminer.java @@ -0,0 +1,485 @@ +/** + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.odpi.specs.runtime.hadoop; + +import org.apache.commons.cli.CommandLine; +import org.apache.commons.cli.GnuParser; +import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Options; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.codehaus.jackson.annotate.JsonIgnore; +import org.codehaus.jackson.map.ObjectMapper; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A tool that generates API conformance tests for Hadoop libraries + */ +public class ApiExaminer { + + private static final Log LOG = LogFactory.getLog(ApiExaminer.class.getName()); + + static private Set<String> unloadableClasses; + + private List<String> errors; + private List<String> warnings; + + static { + unloadableClasses = new HashSet<>(); + unloadableClasses.add("org.apache.hadoop.security.JniBasedUnixGroupsMapping"); + unloadableClasses.add("org.apache.hadoop.security.JniBasedUnixGroupsNetgroupMapping"); + unloadableClasses.add("org.apache.hadoop.io.compress.lz4.Lz4Compressor"); + unloadableClasses.add("org.apache.hadoop.record.compiler.ant.RccTask"); + + } + + public static void main(String[] args) { + Options options = new Options(); + + options.addOption("c", "compare", true, + "Compare against a spec, argument is the json file containing spec"); + options.addOption("h", "help", false, "You're looking at it"); + options.addOption("j", "jar", true, "Jar to examine"); + options.addOption("p", "prepare-spec", true, + "Prepare the spec, argument is the directory to write the spec to"); + + try { + CommandLine cli = new GnuParser().parse(options, args); + + if (cli.hasOption('h')) { + usage(options); + return; + } + + if ((!cli.hasOption('c') && !cli.hasOption('p')) || + (cli.hasOption('c') && cli.hasOption('p'))) { + System.err.println("You must choose either -c or -p"); + usage(options); + return; + } + + if (!cli.hasOption('j')) { + System.err.println("You must specify the jar to prepare or compare"); + usage(options); + return; + } + + String jar = cli.getOptionValue('j'); + ApiExaminer examiner = new ApiExaminer(); + + if (cli.hasOption('c')) { + examiner.compareAgainstStandard(cli.getOptionValue('c'), jar); + } else if (cli.hasOption('p')) { + examiner.prepareExpected(jar, cli.getOptionValue('p')); + } + } catch (Exception e) { + System.err.println("Received exception while processing"); + e.printStackTrace(); + } + } + + private static void usage(Options options) { + HelpFormatter help = new HelpFormatter(); + help.printHelp("api-examiner", options); + + } + + private ApiExaminer() { + } + + private void prepareExpected(String jarFile, String outputDir) throws IOException, + ClassNotFoundException { + JarInfo jarInfo = new JarInfo(jarFile, this); + jarInfo.dumpToFile(new File(outputDir)); + } + + private void compareAgainstStandard(String json, String jarFile) throws IOException, + ClassNotFoundException { + errors = new ArrayList<>(); + warnings = new ArrayList<>(); + JarInfo underTest = new JarInfo(jarFile, this); + JarInfo standard = jarInfoFromFile(new File(json)); + standard.compareAndReport(underTest); + + if (errors.size() > 0) { + System.err.println("Found " + errors.size() + " incompatibilities:"); + for (String error : errors) { + System.err.println(error); + } + } + + if (warnings.size() > 0) { + System.err.println("Found " + warnings.size() + " possible issues: "); + for (String warning : warnings) { + System.err.println(warning); + } + } + + + } + + private JarInfo jarInfoFromFile(File inputFile) throws IOException { + ObjectMapper mapper = new ObjectMapper(); + JarInfo jarInfo = mapper.readValue(inputFile, JarInfo.class); + jarInfo.patchUpClassBackPointers(this); + return jarInfo; + } + + private static class JarInfo { + String name; + String version; + ApiExaminer container; + Map<String, ClassInfo> classes; + + // For use by Jackson + public JarInfo() { + + } + + JarInfo(String jarFile, ApiExaminer container) throws IOException, ClassNotFoundException { + this.container = container; + LOG.info("Processing jar " + jarFile); + File f = new File(jarFile); + Pattern pattern = Pattern.compile("(hadoop-[a-z\\-]+)-([0-9]\\.[0-9]\\.[0-9]).*"); + Matcher matcher = pattern.matcher(f.getName()); + if (!matcher.matches()) { + String msg = "Unable to determine name and version from " + f.getName(); + LOG.error(msg); + throw new RuntimeException(msg); + } + name = matcher.group(1); + version = matcher.group(2); + classes = new HashMap<>(); + + JarFile jar = new JarFile(jarFile); + Enumeration<JarEntry> entries = jar.entries(); + while (entries.hasMoreElements()) { + String name = entries.nextElement().getName(); + if (name.endsWith(".class")) { + name = name.substring(0, name.length() - 6); + name = name.replace('/', '.'); + if (!unloadableClasses.contains(name)) { + LOG.debug("Processing class " + name); + Class<?> clazz = Class.forName(name); + if (clazz.getAnnotation(InterfaceAudience.Public.class) != null && + clazz.getAnnotation(InterfaceStability.Stable.class) != null) { + classes.put(name, new ClassInfo(this, clazz)); + } + } + } + } + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getVersion() { + return version; + } + + public void setVersion(String version) { + this.version = version; + } + + public Map<String, ClassInfo> getClasses() { + return classes; + } + + public void setClasses(Map<String, ClassInfo> classes) { + this.classes = classes; + } + + void compareAndReport(JarInfo underTest) { + Set<ClassInfo> underTestClasses = new HashSet<>(underTest.classes.values()); + for (ClassInfo classInfo : classes.values()) { + if (underTestClasses.contains(classInfo)) { + classInfo.compareAndReport(underTest.classes.get(classInfo.name)); + underTestClasses.remove(classInfo); + } else { + container.errors.add(underTest + " does not contain class " + classInfo); + } + } + + if (underTestClasses.size() > 0) { + for (ClassInfo extra : underTestClasses) { + container.warnings.add(underTest + " contains extra class " + extra); + } + } + } + + void dumpToFile(File outputDir) throws IOException { + File output = new File(outputDir, name + "-" + version + "-api-report.json"); + ObjectMapper mapper = new ObjectMapper(); + mapper.writeValue(output, this); + } + + void patchUpClassBackPointers(ApiExaminer container) { + this.container = container; + for (ClassInfo classInfo : classes.values()) { + classInfo.setJar(this); + classInfo.patchUpBackMethodBackPointers(); + } + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof JarInfo)) return false; + JarInfo that = (JarInfo)other; + return name.equals(that.name) && version.equals(that.version); + } + + @Override + public String toString() { + return name + "-" + version; + } + } + + private static class ClassInfo { + @JsonIgnore JarInfo jar; + String name; + Map<String, MethodInfo> methods; + + // For use by Jackson + public ClassInfo() { + + } + + ClassInfo(JarInfo jar, Class<?> clazz) { + this.jar = jar; + this.name = clazz.getName(); + methods = new HashMap<>(); + + for (Method method : clazz.getMethods()) { + if (method.getDeclaringClass().equals(clazz)) { + LOG.debug("Processing method " + method.getName()); + MethodInfo mi = new MethodInfo(this, method); + methods.put(mi.toString(), mi); + } + } + } + + public JarInfo getJar() { + return jar; + } + + public void setJar(JarInfo jar) { + this.jar = jar; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Map<String, MethodInfo> getMethods() { + return methods; + } + + public void setMethods(Map<String, MethodInfo> methods) { + this.methods = methods; + } + + void compareAndReport(ClassInfo underTest) { + // Make a copy so we can remove them as we match them, making it easy to find additional ones + Set<MethodInfo> underTestMethods = new HashSet<>(underTest.methods.values()); + for (MethodInfo methodInfo : methods.values()) { + if (underTestMethods.contains(methodInfo)) { + methodInfo.compareAndReport(underTest.methods.get(methodInfo.toString())); + underTestMethods.remove(methodInfo); + } else { + jar.container.errors.add(underTest + " does not contain method " + methodInfo); + } + } + + if (underTestMethods.size() > 0) { + for (MethodInfo extra : underTestMethods) { + jar.container.warnings.add(underTest + " contains extra method " + extra); + } + } + } + + void patchUpBackMethodBackPointers() { + for (MethodInfo methodInfo : methods.values()) methodInfo.setContainingClass(this); + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof ClassInfo)) return false; + ClassInfo that = (ClassInfo)other; + return name.equals(that.name); // Classes can be compared just on names + } + + @Override + public int hashCode() { + return name.hashCode(); + } + + @Override + public String toString() { + return jar + " " + name; + } + } + + private static class MethodInfo { + @JsonIgnore ClassInfo containingClass; + String name; + String returnType; + List<String> args; + Set<String> exceptions; + + // For use by Jackson + public MethodInfo() { + + } + + MethodInfo(ClassInfo containingClass, Method method) { + this.containingClass = containingClass; + this.name = method.getName(); + args = new ArrayList<>(); + for (Class<?> argClass : method.getParameterTypes()) { + args.add(argClass.getName()); + } + returnType = method.getReturnType().getName(); + exceptions = new HashSet<>(); + for (Class<?> exception : method.getExceptionTypes()) { + exceptions.add(exception.getName()); + } + } + + public ClassInfo getContainingClass() { + return containingClass; + } + + public void setContainingClass(ClassInfo containingClass) { + this.containingClass = containingClass; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getReturnType() { + return returnType; + } + + public void setReturnType(String returnType) { + this.returnType = returnType; + } + + public List<String> getArgs() { + return args; + } + + public void setArgs(List<String> args) { + this.args = args; + } + + public Set<String> getExceptions() { + return exceptions; + } + + public void setExceptions(Set<String> exceptions) { + this.exceptions = exceptions; + } + + void compareAndReport(MethodInfo underTest) { + // Check to see if they've added or removed exceptions + // Make a copy so I can remove them as I check them off and easily find any that have been + // added. + Set<String> underTestExceptions = new HashSet<>(underTest.exceptions); + for (String exception : exceptions) { + if (underTest.exceptions.contains(exception)) { + underTestExceptions.remove(exception); + } else { + containingClass.jar.container.warnings.add(underTest.containingClass.jar + " " + + underTest.containingClass + "." + name + " removes exception " + exception); + } + } + if (underTestExceptions.size() > 0) { + for (String underTestException : underTest.exceptions) { + containingClass.jar.container.warnings.add(underTest.containingClass.jar + " " + + underTest.containingClass + "." + name + " adds exception " + underTestException); + } + } + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof MethodInfo)) return false; + MethodInfo that = (MethodInfo)other; + + return containingClass.equals(that.containingClass) && name.equals(that.name) && + returnType.equals(that.returnType) && args.equals(that.args); + } + + @Override + public int hashCode() { + return ((containingClass.hashCode() * 31 + name.hashCode()) * 31 + returnType.hashCode()) * 31 + + args.hashCode(); + } + + @Override + public String toString() { + StringBuilder buf = new StringBuilder(returnType) + .append(" ") + .append(name) + .append('('); + boolean first = true; + for (String arg : args) { + if (first) first = false; + else buf.append(", "); + buf.append(arg); + } + buf.append(")"); + if (exceptions.size() > 0) { + buf.append(" throws "); + first = true; + for (String exception : exceptions) { + if (first) first = false; + else buf.append(", "); + buf.append(exception); + } + } + return buf.toString(); + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/5e342c45/bigtop-tests/smoke-tests/odpi-runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/odpi-runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java b/bigtop-tests/smoke-tests/odpi-runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java new file mode 100644 index 0000000..4110d5d --- /dev/null +++ b/bigtop-tests/smoke-tests/odpi-runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.odpi.specs.runtime.hive; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.io.IntWritable; +import org.apache.hadoop.io.Text; +import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.mapreduce.Job; +import org.apache.hadoop.mapreduce.Mapper; +import org.apache.hadoop.mapreduce.Reducer; +import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.hadoop.util.Tool; +import org.apache.hadoop.util.ToolRunner; +import org.apache.hive.hcatalog.data.DefaultHCatRecord; +import org.apache.hive.hcatalog.data.HCatRecord; +import org.apache.hive.hcatalog.data.schema.HCatSchema; +import org.apache.hive.hcatalog.data.schema.HCatSchemaUtils; +import org.apache.hive.hcatalog.mapreduce.HCatInputFormat; +import org.apache.hive.hcatalog.mapreduce.HCatOutputFormat; +import org.apache.hive.hcatalog.mapreduce.OutputJobInfo; + +import java.io.IOException; +import java.net.URI; +import java.util.StringTokenizer; + +public class HCatalogMR extends Configured implements Tool { + private final static String INPUT_SCHEMA = "odpi.test.hcat.schema.input"; + private final static String OUTPUT_SCHEMA = "odpi.test.hcat.schema.output"; + + @Override + public int run(String[] args) throws Exception { + String inputTable = null; + String outputTable = null; + String inputSchemaStr = null; + String outputSchemaStr = null; + for(int i = 0; i < args.length; i++){ + if(args[i].equalsIgnoreCase("-it")){ + inputTable = args[i+1]; + }else if(args[i].equalsIgnoreCase("-ot")){ + outputTable = args[i+1]; + }else if(args[i].equalsIgnoreCase("-is")){ + inputSchemaStr = args[i+1]; + }else if(args[i].equalsIgnoreCase("-os")){ + outputSchemaStr = args[i+1]; + } + } + + Configuration conf = getConf(); + args = new GenericOptionsParser(conf, args).getRemainingArgs(); + + conf.set(INPUT_SCHEMA, inputSchemaStr); + conf.set(OUTPUT_SCHEMA, outputSchemaStr); + + Job job = new Job(conf, "odpi_hcat_test"); + HCatInputFormat.setInput(job, "default", inputTable); + + job.setInputFormatClass(HCatInputFormat.class); + job.setJarByClass(HCatalogMR.class); + job.setMapperClass(Map.class); + job.setReducerClass(Reduce.class); + job.setMapOutputKeyClass(Text.class); + job.setMapOutputValueClass(IntWritable.class); + job.setOutputKeyClass(WritableComparable.class); + job.setOutputValueClass(HCatRecord.class); + HCatOutputFormat.setOutput(job, OutputJobInfo.create("default", outputTable, null)); + HCatOutputFormat.setSchema(job, HCatSchemaUtils.getHCatSchema(outputSchemaStr)); + job.setOutputFormatClass(HCatOutputFormat.class); + + return job.waitForCompletion(true) ? 0 : 1; + + + } + public static class Map extends Mapper<WritableComparable, + HCatRecord, Text, IntWritable> { + private final static IntWritable one = new IntWritable(1); + private Text word = new Text(); + private HCatSchema inputSchema = null; + + @Override + protected void map(WritableComparable key, HCatRecord value, Context context) + throws IOException, InterruptedException { + if (inputSchema == null) { + inputSchema = + HCatSchemaUtils.getHCatSchema(context.getConfiguration().get(INPUT_SCHEMA)); + } + String line = value.getString("line", inputSchema); + StringTokenizer tokenizer = new StringTokenizer(line); + while (tokenizer.hasMoreTokens()) { + word.set(tokenizer.nextToken()); + context.write(word, one); + } + } + } + + public static class Reduce extends Reducer<Text, IntWritable, WritableComparable, HCatRecord> { + private HCatSchema outputSchema = null; + + @Override + protected void reduce(Text key, Iterable<IntWritable> values, Context context) throws + IOException, InterruptedException { + if (outputSchema == null) { + outputSchema = + HCatSchemaUtils.getHCatSchema(context.getConfiguration().get(OUTPUT_SCHEMA)); + } + int sum = 0; + for (IntWritable i : values) { + sum += i.get(); + } + HCatRecord output = new DefaultHCatRecord(2); + output.set("word", outputSchema, key); + output.set("count", outputSchema, sum); + context.write(null, output); + } + } + + public static void main(String[] args) throws Exception { + int exitCode = ToolRunner.run(new HCatalogMR(), args); + System.exit(exitCode); + } + } http://git-wip-us.apache.org/repos/asf/bigtop/blob/5e342c45/bigtop-tests/smoke-tests/odpi-runtime/src/main/resources/api-examiner-prep.sh ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/odpi-runtime/src/main/resources/api-examiner-prep.sh b/bigtop-tests/smoke-tests/odpi-runtime/src/main/resources/api-examiner-prep.sh new file mode 100755 index 0000000..8c9ab5e --- /dev/null +++ b/bigtop-tests/smoke-tests/odpi-runtime/src/main/resources/api-examiner-prep.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +############################################################################ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +############################################################################ + +############################################################################ +# This script is used to generate the hadoop-*-api.report.json files in the +# test/resources directory. To use it, you will first need to download an +# Apache binary distribution of Hadoop and set APACHE_HADOOP_DIR to the +# directory where you untar that distribution. You will then need to set +# BIGTTOP_HOME to the directory where your bigtop source is located. Then +# run this script for each of the jars you want to generate a report for. +# The arguments passed to this script should be -p <outputdir> -j <jarfile> +# where outputdir is the directory you'd like to write the report to and +# jarfile is the full path of the jar to generate the report for. Reports +# should be generated for the following jars: hadoop-common, hadoop-hdfs, +# hadoop-yarn-common, hadoop-yarn-client, hadoop-yarn-api, and +# hadoop-mapreduce-client-core +# +# Example usage: +# export APACHE_HADOOP_DIR=/tmp/hadoop-2.7.3 +# export BIGTOP_HOME=/home/me/git/bigtop +# $BIGTOP_HOME/bigtop-tests/spec-tests/runtime/src/main/resources/api-examiner.sh -j $HADOOP_HOME/share/hadoop/common/hadoop-common-2.7.3.jar -p $BIGTOP_HOME/bigtop-tests/spec-tests/runtime/src/test/resources +# +# The resulting reports should be committed to git. This script only needs +# to be run once per ODPi release. +############################################################################ + + +if [ "x${APACHE_HADOOP_DIR}" = "x" ] +then + echo "You must set APACHE_HADOOP_DIR to the directory you have placed the Apache Hadoop binary distribution in" + exit 1 +fi + +if [ "x${BIGTOP_HOME}" = "x" ] +then + echo "You must set BIGTOP_HOME to the root directory for your bigtop source" + exit 1 +fi + +for jar in `find $BIGTOP_HOME/bigtop-tests/spec-tests/runtime/build/libs/ -name \*.jar` +do + CLASSPATH=$CLASSPATH:$jar +done + +for jar in `find $APACHE_HADOOP_DIR -name \*.jar` +do + CLASSPATH=$CLASSPATH:$jar +done + +java -cp $CLASSPATH org.odpi.specs.runtime.hadoop.ApiExaminer $@ + http://git-wip-us.apache.org/repos/asf/bigtop/blob/5e342c45/bigtop-tests/smoke-tests/odpi-runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/odpi-runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy b/bigtop-tests/smoke-tests/odpi-runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy new file mode 100644 index 0000000..bc2a3b2 --- /dev/null +++ b/bigtop-tests/smoke-tests/odpi-runtime/src/test/groovy/org/odpi/specs/runtime/TestSpecsRuntime.groovy @@ -0,0 +1,275 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.odpi.specs.runtime + +import groovy.io.FileType +import org.junit.Assert +import org.apache.bigtop.itest.shell.* +import org.junit.Test +import org.junit.runner.RunWith +import org.junit.runners.Parameterized +import org.junit.runners.Parameterized.Parameters + +import java.util.regex.Matcher +import java.util.regex.Pattern + +/** + * Check all expected environment + * Tests are constructed dynamically, using external DSL to define + * - test name + * - test type + * - command to execute the test + * - expected pattern of the output + */ +@RunWith(Parameterized.class) +public class TestSpecsRuntime { + private String testName + private String type + private Map arguments + + private static ENV = System.getenv() + + @Parameters(name="{0}") + public static Collection<Object[]> allTests() { + List<Object[]> specs = []; + + config.specs.tests.each { test -> + specs.add([test.value.name, test.value.type, test.value.arguments] as Object[]) + } + return specs + } + + public TestSpecsRuntime (String testName, String type, Map arguments) { + this.testName = testName + this.type = type + this.arguments = arguments + } + + public static final String testsList = System.properties['test.resources.dir'] ?: + "${System.properties['buildDir']}/resources/test" + def final static config = new ConfigSlurper().parse(new URL("file:${getTestConfigName()}")) + + private static String getTestConfigName() { + return "$testsList/testRuntimeSpecConf.groovy"; + } + + private Map getEnvMap(String command) { + def envMap = [:] + Shell sh = new Shell() + def envvars = sh.exec(command).getOut() + if (sh.getRet() == 0) { + envvars.each { + def match = it =~ /(?<variable>[^=]+)='(?<value>[^']+)'$/ + if ( match.matches() ) { + envMap[match.group('variable')] = match.group('value') + } + } + } + return envMap + } + + private String getEnv(String name, String cmd) { + String value = ENV[name] + if (value == null) { + value = getEnvMap(cmd)[name] + } + return value + } + + @Test + public void testAll() { + switch (type) { + case 'shell': + Shell sh = new Shell() + def output = sh.exec(arguments['command']).getOut().join("\n") + int actualResult = sh.getRet() + int expectedResult = arguments['expectedResult'] ? arguments['expectedResult'] : 0 // use 0 as default success code + Assert.assertTrue("${testName} fail: ${arguments['message']} - '${arguments['command']}' returned ${actualResult} instead of ${expectedResult}", + actualResult == expectedResult) + break + + case 'envdir': + def var = arguments['variable'] + def isPathRelative = arguments['relative'] + def pathString = getEnv(var, arguments['envcmd']) + Assert.assertTrue("${testName} fail: environment variable ${var} does not exist", pathString != null ) + + if ( arguments['pattern'] ) { + Assert.assertTrue("${testName} fail: $pathString doesn't contain expected pattern", + pathString ==~ /${arguments['pattern']}/) + } + + def pathFile = new File(pathString) + if ( isPathRelative ) { + Assert.assertFalse("${testName} fail: ${pathString} is not relative", pathFile.isAbsolute() ) + } else { + if (!arguments['donotcheckexistance']) { + Assert.assertTrue("${testName} fail: ${pathString} does not exist", pathFile.exists() ) + Assert.assertTrue("${testName} fail: ${pathString} is not directory", pathFile.isDirectory() ) + } + } + break + + case 'dirstruct': + def expectedFiles = [] + new File("${testsList}", "${arguments['referenceList']}").eachLine { line -> + expectedFiles << ~line + } + def baseDirEnv = getEnv(arguments['baseDirEnv'], arguments['envcmd']) + Assert.assertNotNull("${baseDirEnv} has to be set for the test to continue", + baseDirEnv) + def root = new File(baseDirEnv) + def actualFiles = [] + def missingFiles = [] + if ( ! root.exists() ) { + Assert.assertFail("${testName} fail: ${baseDirEnv} does not exist!"); + } + + root.eachFileRecurse(FileType.ANY) { file -> + def relPath = new File( root.toURI().relativize( file.toURI() ).toString() ).path + actualFiles << relPath + } + + expectedFiles.each { wantFile -> + def ok = false + for (def x : actualFiles) { + if (actualFiles =~ wantFile) { + ok = true + break + } + } + if (!ok) { + missingFiles << wantFile + } + } + + Assert.assertTrue("${testName} fail: Directory structure for ${baseDirEnv} does not match reference. Missing files: ${missingFiles} ", + missingFiles.size() == 0) + break + + case 'dircontent': + def expectedFiles = [] + new File("${testsList}", "${arguments['referenceList']}").eachLine { line -> + expectedFiles << ~line + } + + def baseDir = getEnv(arguments['baseDirEnv'], arguments['envcmd']) + def subDir = arguments['subDir'] + if (!subDir && arguments['subDirEnv']) { + subDir = getEnv(arguments['subDirEnv'], arguments['envcmd']) + } + + def dir = null + if (subDir) { + dir = new File(baseDir, subDir) + } else { + dir = new File(baseDir) + } + Assert.assertNotNull("Directory has to be set for the test to continue", dir) + + def actualFiles = [] + if (dir.exists()) { + dir.eachFile FileType.FILES, { file -> + def relPath = new File( dir.toURI().relativize( file.toURI() ).toString() ).path + actualFiles << relPath + } + } + + def missingList = [] + for (def wantFile : expectedFiles) { + def ok = false + for (def haveFile : actualFiles) { + if (haveFile =~ wantFile) { + ok = true + break + } + } + if (! ok) { + missingList << wantFile + } + } + + def extraList = [] + for (def haveFile : actualFiles) { + def ok = false + for (def wantFile : expectedFiles) { + if (haveFile =~ wantFile) { + ok = true + break + } + } + if (! ok) { + extraList << haveFile + } + } + + def commonFiles = actualFiles.intersect(expectedFiles) + Assert.assertTrue("${testName} fail: Directory content for ${dir.path} does not match reference. Missing files: ${missingList}. Extra files: ${extraList}", + missingList.size() == 0 && extraList.size() == 0) + break + case 'hadoop_tools': + def toolsPathStr = getEnv("HADOOP_TOOLS_PATH", "hadoop envvars") + Assert.assertNotNull("${testName} fail: HADOOP_TOOLS_PATH environment variable should be set", toolsPathStr) + + def toolsPath = new File(toolsPathStr) + Assert.assertTrue("${testName} fail: HADOOP_TOOLS_PATH must be an absolute path.", toolsPath.isAbsolute()) + + Shell sh = new Shell() + def classPath = sh.exec("hadoop classpath").getOut().join("\n") + Assert.assertTrue("${testName} fail: Failed to retrieve hadoop's classpath", sh.getRet()==0) + + Assert.assertFalse("${testName} fail: The enire '${toolsPath}' path should not be included in the hadoop's classpath", + classPath.split(File.pathSeparator).any { + new File(it).getCanonicalPath() =~ /^${toolsPath}\/?\*/ + } + ) + break + case 'api_examination': + def basedir = getEnv(arguments['baseDirEnv'], arguments['envcmd']) + def libdir = getEnv(arguments['libDir'], arguments['envcmd']) + + def dir = new File(basedir + "/" + libdir) + Assert.assertTrue("Expected " + dir.getPath() + " to be a directory", dir.isDirectory()) + def pattern = Pattern.compile(arguments['jar'] + "-[0-9]+.*\\.jar") + def String[] jars = dir.list(new FilenameFilter() { + @Override + boolean accept(File d, String name) { + Matcher matcher = pattern.matcher(name) + return (matcher.matches() && !name.contains("test")) + } + }) + Assert.assertEquals("Expected only one jar, but got " + jars.join(", "), 1, jars.length) + def jar = dir.getAbsolutePath() + "/" + jars[0] + + def examinerJar = System.properties['odpi.test.hive.hcat.job.jar'] + def resourceFile = System.properties['test.resources.dir']+ "/" + arguments['resourceFile'] + Shell sh = new Shell() + def results = sh.exec("hadoop jar " + examinerJar + " org.odpi.specs.runtime.hadoop.ApiExaminer -c " + resourceFile + " -j " + jar).getErr() + int rc = sh.getRet() + Assert.assertEquals("Expected command to succeed, but got return code " + rc, 0, rc) + if (results.size() > 0) { + System.out.println("Received report for jar " + arguments['jar'] + results.join("\n")) + } + break; + + + default: + break + } + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/5e342c45/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/HiveHelper.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/HiveHelper.java b/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/HiveHelper.java new file mode 100644 index 0000000..3e56224 --- /dev/null +++ b/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/HiveHelper.java @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.odpi.specs.runtime.hive; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DefaultExecuteResultHandler; +import org.apache.commons.exec.DefaultExecutor; +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.exec.ExecuteWatchdog; +import org.apache.commons.exec.Executor; +import org.apache.commons.exec.PumpStreamHandler; +import org.apache.commons.exec.environment.EnvironmentUtils; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class HiveHelper { + + private static final Log LOG = LogFactory.getLog(HiveHelper.class.getName()); + + public static Map<String, String> execCommand(CommandLine commandline) { + return execCommand(commandline, null); + } + + public static Map<String, String> execCommand(CommandLine commandline, + Map<String, String> envVars) { + + System.out.println("Executing command:"); + System.out.println(commandline.toString()); + Map<String, String> env = null; + Map<String, String> entry = new HashMap<String, String>(); + try { + env = EnvironmentUtils.getProcEnvironment(); + } catch (IOException e1) { + // TODO Auto-generated catch block + LOG.debug("Failed to get process environment: "+ e1.getMessage()); + e1.printStackTrace(); + } + if (envVars != null) { + for (String key : envVars.keySet()) { + env.put(key, envVars.get(key)); + } + } + + DefaultExecuteResultHandler resultHandler = new DefaultExecuteResultHandler(); + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + PumpStreamHandler streamHandler = new PumpStreamHandler(outputStream); + ExecuteWatchdog watchdog = new ExecuteWatchdog(60*10000); + Executor executor = new DefaultExecutor(); + executor.setExitValue(1); + executor.setWatchdog(watchdog); + executor.setStreamHandler(streamHandler); + try { + executor.execute(commandline, env, resultHandler); + } catch (ExecuteException e) { + // TODO Auto-generated catch block + LOG.debug("Failed to execute command with exit value: "+ String.valueOf(resultHandler.getExitValue())); + LOG.debug("outputStream: "+ outputStream.toString()); + entry.put("exitValue", String.valueOf(resultHandler.getExitValue())); + entry.put("outputStream", outputStream.toString() + e.getMessage()); + e.printStackTrace(); + return entry; + } catch (IOException e) { + // TODO Auto-generated catch block + LOG.debug("Failed to execute command with exit value: "+ String.valueOf(resultHandler.getExitValue())); + LOG.debug("outputStream: "+ outputStream.toString()); + entry.put("exitValue", String.valueOf(resultHandler.getExitValue())); + entry.put("outputStream", outputStream.toString() + e.getMessage()); + e.printStackTrace(); + return entry; + } + + try { + resultHandler.waitFor(); + /*System.out.println("Command output: "+outputStream.toString());*/ + entry.put("exitValue", String.valueOf(resultHandler.getExitValue())); + entry.put("outputStream", outputStream.toString()); + return entry; + } catch (InterruptedException e) { + // TODO Auto-generated catch block + /*System.out.println("Command output: "+outputStream.toString());*/ + LOG.debug("exitValue: "+ String.valueOf(resultHandler.getExitValue())); + LOG.debug("outputStream: "+ outputStream.toString()); + entry.put("exitValue", String.valueOf(resultHandler.getExitValue())); + entry.put("outputStream", outputStream.toString()); + e.printStackTrace(); + return entry; + } + } + + protected static String getProperty(String property, String description) { + String val = System.getProperty(property); + if (val == null) { + throw new RuntimeException("You must set the property " + property + " with " + + description); + } + LOG.debug(description + " is " + val); + return val; + } + + +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/5e342c45/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/JdbcConnector.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/JdbcConnector.java b/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/JdbcConnector.java new file mode 100644 index 0000000..7512dab --- /dev/null +++ b/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/JdbcConnector.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.odpi.specs.runtime.hive; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.util.Properties; + +public class JdbcConnector { + private static final Log LOG = LogFactory.getLog(JdbcConnector.class.getName()); + + protected static final String URL = "odpi.test.hive.jdbc.url"; + protected static final String USER = "odpi.test.hive.jdbc.user"; + protected static final String PASSWD = "odpi.test.hive.jdbc.password"; + protected static final String LOCATION = "odpi.test.hive.location"; + protected static final String METASTORE_URL = "odpi.test.hive.metastore.url"; + protected static final String TEST_THRIFT = "odpi.test.hive.thrift.test"; + protected static final String TEST_HCATALOG = "odpi.test.hive.hcatalog.test"; + protected static final String HIVE_CONF_DIR = "odpi.test.hive.conf.dir"; + protected static final String HADOOP_CONF_DIR = "odpi.test.hadoop.conf.dir"; + + protected static Connection conn; + + @BeforeClass + public static void connectToJdbc() throws SQLException { + // Assume they've put the URL for the JDBC driver in an environment variable. + String jdbcUrl = getProperty(URL, "the JDBC URL"); + String jdbcUser = getProperty(USER, "the JDBC user name"); + String jdbcPasswd = getProperty(PASSWD, "the JDBC password"); + + Properties props = new Properties(); + props.put("user", jdbcUser); + if (!jdbcPasswd.equals("")) props.put("password", jdbcPasswd); + conn = DriverManager.getConnection(jdbcUrl, props); + } + + @AfterClass + public static void closeJdbc() throws SQLException { + if (conn != null) conn.close(); + } + + protected static String getProperty(String property, String description) { + String val = System.getProperty(property); + if (val == null) { + throw new RuntimeException("You must set the property " + property + " with " + + description); + } + LOG.debug(description + " is " + val); + return val; + } + + protected static boolean testActive(String property, String description) { + String val = System.getProperty(property, "true"); + LOG.debug(description + " is " + val); + return Boolean.valueOf(val); + } + +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/5e342c45/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestBeeline.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestBeeline.java b/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestBeeline.java new file mode 100644 index 0000000..578621a --- /dev/null +++ b/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestBeeline.java @@ -0,0 +1,201 @@ +package org.odpi.specs.runtime.hive; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; +import java.io.FileNotFoundException; +import java.io.PrintWriter; +import java.util.Map; + +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class TestBeeline { + + public static final Log LOG = LogFactory.getLog(TestBeeline.class.getName()); + + private static final String URL = "odpi.test.hive.jdbc.url"; + private static final String USER = "odpi.test.hive.jdbc.user"; + private static final String PASSWD = "odpi.test.hive.jdbc.password"; + + private static Map<String, String> results; + private static String beelineUrl; + private static String beelineUser; + private static String beelinePasswd; + + //creating beeline base command with username and password as per inputs + private static CommandLine beelineBaseCommand = new CommandLine("beeline"); + + @BeforeClass + public static void initialSetup(){ + TestBeeline.beelineUrl = System.getProperty(URL); + TestBeeline.beelineUser = System.getProperty(USER); + TestBeeline.beelinePasswd =System.getProperty(PASSWD); + + if (beelineUser != null && beelineUser != "" && beelinePasswd != null && beelinePasswd != "") + { + beelineBaseCommand.addArgument("-u").addArgument(beelineUrl).addArgument("-n").addArgument(beelineUser).addArgument("-p").addArgument(beelinePasswd); + } + else if (beelineUser != null && beelineUser != "") + { + beelineBaseCommand.addArgument("-u").addArgument(beelineUrl).addArgument("-n").addArgument(beelineUser); + } + else { + beelineBaseCommand.addArgument("-u").addArgument(beelineUrl); + } + LOG.info("URL is " + beelineUrl); + LOG.info("User is " + beelineUser); + LOG.info("Passwd is " + beelinePasswd); + LOG.info("Passwd is null " + (beelinePasswd == null)); + } + + @Test + public void checkBeeline() { + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand)); + String consoleMsg = results.get("outputStream").toLowerCase(); + Assert.assertEquals("beeline -u FAILED. \n" +results.get("outputStream"), true, consoleMsg.contains("connecting to "+beelineUrl.toLowerCase()) && !consoleMsg.contains("error") && !consoleMsg.contains("exception")); + } + + @Test + public void checkBeelineConnect(){ + try(PrintWriter out = new PrintWriter("connect.url")){ out.println("!connect " + beelineUrl+" "+beelineUser+" "+beelinePasswd); out.println("!quit"); } + catch (FileNotFoundException e1) { + e1.printStackTrace(); + } + results = HiveHelper.execCommand(new CommandLine("/bin/sh").addArgument("-c").addArgument("beeline -f connect.url",false)); + String consoleMsg = results.get("outputStream").toLowerCase(); + Assert.assertEquals("beeline !connect FAILED. \n" +results.get("outputStream"), true, consoleMsg.contains("connecting to "+beelineUrl.toLowerCase()) && !consoleMsg.contains("error") && !consoleMsg.contains("exception") ); + } + + @Test + public void checkBeelineHelp(){ + results = HiveHelper.execCommand(new CommandLine("beeline").addArgument("--help")); + String consoleMsg = results.get("outputStream").toLowerCase(); + Assert.assertEquals("beeline --help FAILED. \n" +results.get("outputStream"), true, consoleMsg.contains("display this message" ) && consoleMsg.contains("usage: java org.apache.hive.cli.beeline.beeline") && !consoleMsg.contains("exception")); + } + + @Test + public void checkBeelineQueryExecFromCmdLine(){ + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-e").addArgument("SHOW DATABASES;")); + if(!results.get("outputStream").contains("odpi_runtime_hive")){ + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-e").addArgument("CREATE DATABASE odpi_runtime_hive;")); + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-e").addArgument("SHOW DATABASES;")); + }else{ + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-e").addArgument("DROP DATABASE odpi_runtime_hive;")); + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-e").addArgument("CREATE DATABASE odpi_runtime_hive;")); + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-e").addArgument("SHOW DATABASES;")); + } + String consoleMsg = results.get("outputStream").toLowerCase(); + Assert.assertEquals("beeline -e FAILED. \n" +results.get("outputStream"), true, consoleMsg.contains("odpi_runtime_hive" ) && !consoleMsg.contains("error") && !consoleMsg.contains("exception")); + HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-e").addArgument("DROP DATABASE odpi_runtime_hive")); + } + + @Test + public void checkBeelineQueryExecFromFile() throws FileNotFoundException{ + + try(PrintWriter out = new PrintWriter("beeline-f1.sql")){ out.println("SHOW DATABASES;"); } + try(PrintWriter out = new PrintWriter("beeline-f2.sql")){ out.println("CREATE DATABASE odpi_runtime_hive;"); } + try(PrintWriter out = new PrintWriter("beeline-f3.sql")){ out.println("DROP DATABASE odpi_runtime_hive;"); out.println("CREATE DATABASE odpi_runtime_hive;"); } + try(PrintWriter out = new PrintWriter("beeline-f4.sql")){ out.println("DROP DATABASE odpi_runtime_hive;"); } + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-f").addArgument("beeline-f1.sql",false)); + + if(!results.get("outputStream").contains("odpi_runtime_hive")){ + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-f").addArgument("beeline-f2.sql",false)); + }else{ + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-f").addArgument("beeline-f3.sql",false)); + } + + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-f").addArgument("beeline-f1.sql",false)); + + String consoleMsg = results.get("outputStream").toLowerCase(); + Assert.assertEquals("beeline -f FAILED. \n" +results.get("outputStream"), true, consoleMsg.contains("odpi_runtime_hive" ) && !consoleMsg.contains("error") && !consoleMsg.contains("exception")); + HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-f").addArgument("beeline-f4.sql",false)); + } + + @Test + public void checkBeelineInitFile() throws FileNotFoundException{ + + try(PrintWriter out = new PrintWriter("beeline-i1.sql")){ out.println("SHOW DATABASES;"); } + try(PrintWriter out = new PrintWriter("beeline-i2.sql")){ out.println("CREATE DATABASE odpi_runtime_beeline_init;"); } + try(PrintWriter out = new PrintWriter("beeline-i3.sql")){ out.println("DROP DATABASE odpi_runtime_beeline_init;"); out.println("CREATE DATABASE odpi_runtime_beeline_init;"); } + try(PrintWriter out = new PrintWriter("beeline-i4.sql")){ out.println("DROP DATABASE odpi_runtime_beeline_init;"); } + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-i").addArgument("beeline-i1.sql",false)); + + if(!results.get("outputStream").contains("odpi_runtime_beeline_init")){ + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-i").addArgument("beeline-i2.sql",false)); + }else{ + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-i").addArgument("beeline-i3.sql",false)); + } + + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-i").addArgument("beeline-i1.sql",false)); + String consoleMsg = results.get("outputStream").toLowerCase(); + Assert.assertEquals("beeline -i FAILED. \n" +results.get("outputStream"), true, consoleMsg.contains("odpi_runtime_beeline_init") && !consoleMsg.contains("error") && !consoleMsg.contains("exception")); + HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("-i").addArgument("beeline-i4.sql",false)); + } + + @Test + public void checkBeelineHiveVar() throws FileNotFoundException{ + + try(PrintWriter out = new PrintWriter("beeline-hv1.sql")){ out.println("SHOW DATABASES;"); } + try(PrintWriter out = new PrintWriter("beeline-hv2.sql")){ out.println("CREATE DATABASE ${db};"); } + try(PrintWriter out = new PrintWriter("beeline-hv3.sql")){ out.println("DROP DATABASE ${db};"); out.println("CREATE DATABASE ${db};"); } + try(PrintWriter out = new PrintWriter("beeline-hv4.sql")){ out.println("DROP DATABASE ${db};"); } + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("--hivevar").addArgument("db=odpi_runtime_beeline_hivevar").addArgument("-i").addArgument("beeline-hv1.sql",false)); + + if(!results.get("outputStream").contains("odpi_runtime_beeline_hivevar")){ + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("--hivevar").addArgument("db=odpi_runtime_beeline_hivevar").addArgument("-i").addArgument("beeline-hv2.sql",false)); + }else{ + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("--hivevar").addArgument("db=odpi_runtime_beeline_hivevar").addArgument("-i").addArgument("beeline-hv3.sql",false)); + } + + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("--hivevar").addArgument("db=odpi_runtime_beeline_hivevar").addArgument("-i").addArgument("beeline-hv1.sql",false)); + String consoleMsg = results.get("outputStream").toLowerCase(); + Assert.assertEquals("beeline --hivevar FAILED. \n" +results.get("outputStream"), true, consoleMsg.contains("odpi_runtime_beeline_hivevar") && !consoleMsg.contains("error") && !consoleMsg.contains("exception")); + HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("--hivevar").addArgument("db=odpi_runtime_beeline_hivevar").addArgument("-i").addArgument("beeline-hv4.sql",false)); + } + + @Test + public void checkBeelineFastConnect(){ + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("--fastConnect=false")); + String consoleMsg = results.get("outputStream").toLowerCase(); + Assert.assertEquals("beeline --fastConnect FAILED. \n" +results.get("outputStream"), true, consoleMsg.contains("set fastconnect to true to skip")); + } + + @Test + public void checkBeelineVerbose(){ + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("--verbose=true")); + String consoleMsg = results.get("outputStream").toLowerCase(); + Assert.assertEquals("beeline --verbose FAILED." +results.get("outputStream"), true, consoleMsg.contains("issuing: !connect jdbc:hive2:") && !consoleMsg.contains("error") && !consoleMsg.contains("exception")); + } + + @Test + public void checkBeelineShowHeader(){ + results = HiveHelper.execCommand(new CommandLine(beelineBaseCommand).addArgument("--showHeader=false").addArgument("-e").addArgument("SHOW DATABASES;")); + String consoleMsg = results.get("outputStream").toLowerCase(); + Assert.assertEquals("beeline --showHeader FAILED. \n" +results.get("outputStream"), true, consoleMsg.contains("default")&&!consoleMsg.contains("database_name") && !consoleMsg.contains("error") && !consoleMsg.contains("exception")); + } + + @AfterClass + public static void cleanup() throws FileNotFoundException { + results = HiveHelper.execCommand(new CommandLine("/bin/sh").addArgument("-c").addArgument("rm -rf beeline*.sql", false)); + results = HiveHelper.execCommand(new CommandLine("/bin/sh").addArgument("-c").addArgument("rm -rf connect.url", false)); + } +} http://git-wip-us.apache.org/repos/asf/bigtop/blob/5e342c45/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestCLI.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestCLI.java b/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestCLI.java new file mode 100644 index 0000000..2b70909 --- /dev/null +++ b/bigtop-tests/smoke-tests/odpi-runtime/src/test/java/org/odpi/specs/runtime/hive/TestCLI.java @@ -0,0 +1,213 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.odpi.specs.runtime.hive; + +import java.io.FileNotFoundException; +import java.io.PrintWriter; +import java.util.Map; + +import org.apache.commons.exec.CommandLine; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.AfterClass; +import org.junit.Assert; + +public class TestCLI { + + static Map<String, String> results; + static String db = "javax.jdo.option.ConnectionURL=jdbc:derby:;databaseName=odpi_metastore_db;create=true"; + + @BeforeClass + public static void setup(){ + + results = HiveHelper.execCommand(new CommandLine("which").addArgument("hive")); + Assert.assertEquals("Hive is not in the current path.", 0, Integer.parseInt(results.get("exitValue"))); + } + + @Test + public void help(){ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-H")); + //LOG.info(results.get("exitValue")); + Assert.assertEquals("Error in executing 'hive -H'", 2, Integer.parseInt(results.get("exitValue"))); + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("--help")); + Assert.assertEquals("Error in executing 'hive --help'", 0, Integer.parseInt(results.get("exitValue"))); + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-U")); + Assert.assertEquals("Unrecognized option should exit 1.", 1, Integer.parseInt(results.get("exitValue"))); + } + + @Test + public void sqlFromCmdLine(){ + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("SHOW DATABASES").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("SHOW DATABASES command failed to execute.", 0, Integer.parseInt(results.get("exitValue"))); + if(!results.get("outputStream").contains("odpi_runtime_hive")){ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("CREATE DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("Could not create database odpi_runtime_hive.", 0, Integer.parseInt(results.get("exitValue"))); + }else{ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("DROP DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("CREATE DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("Could not create database odpi_runtime_hive.", 0, Integer.parseInt(results.get("exitValue"))); + } + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("DROP DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + } + + @Test + public void sqlFromFiles() throws FileNotFoundException{ + try(PrintWriter out = new PrintWriter("hive-f1.sql")){ out.println("SHOW DATABASES;"); } + try(PrintWriter out = new PrintWriter("hive-f2.sql")){ out.println("CREATE DATABASE odpi_runtime_hive;"); } + try(PrintWriter out = new PrintWriter("hive-f3.sql")){ out.println("DROP DATABASE odpi_runtime_hive;"); out.println("CREATE DATABASE odpi_runtime_hive;"); } + try(PrintWriter out = new PrintWriter("hive-f4.sql")){ out.println("DROP DATABASE odpi_runtime_hive;"); } + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-f").addArgument("hive-f1.sql").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("SHOW DATABASES command failed to execute.", 0, Integer.parseInt(results.get("exitValue"))); + if(!results.get("outputStream").contains("odpi_runtime_hive")){ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-f").addArgument("hive-f2.sql").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("Could not create database odpi_runtime_hive.", 0, Integer.parseInt(results.get("exitValue"))); + }else{ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-f").addArgument("hive-f3.sql").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("Could not create database odpi_runtime_hive.", 0, Integer.parseInt(results.get("exitValue"))); + } + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-f").addArgument("hive-f4.sql").addArgument("--hiveconf").addArgument(db)); + } + + @Test + public void silent() { + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("SHOW DATABASES").addArgument("-S").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("-S option did not work.", new Boolean(false), results.get("outputStream").contains("Time taken:")); + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("SHOW DATABASES").addArgument("--silent").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("--silent option did not work.", new Boolean(false), results.get("outputStream").contains("Time taken:")); + } + + @Test + public void verbose(){ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("SHOW DATABASES").addArgument("-v").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("-v option did not work.", new Boolean(true), results.get("outputStream").contains("SHOW DATABASES")); + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("SHOW DATABASES").addArgument("--verbose").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("--verbose option did not work.", new Boolean(true), results.get("outputStream").contains("SHOW DATABASES")); + } + + @Test + public void initialization() throws FileNotFoundException{ + try(PrintWriter out = new PrintWriter("hive-init1.sql")){ out.println("CREATE DATABASE odpi_runtime_hive;"); } + try(PrintWriter out = new PrintWriter("hive-init2.sql")){ out.println("DROP DATABASE odpi_runtime_hive;"); out.println("CREATE DATABASE odpi_runtime_hive;"); } + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("SHOW DATABASES").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("SHOW DATABASES command failed to execute.", 0, Integer.parseInt(results.get("exitValue"))); + if(!results.get("outputStream").contains("odpi_runtime_hive")){ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-i").addArgument("hive-init1.sql").addArgument("-e").addArgument("SHOW DATABASES").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("Could not create database odpi_runtime_hive using the init -i option.", 0, Integer.parseInt(results.get("exitValue"))); + Assert.assertEquals("Could not create database odpi_runtime_hive using the init -i option.", true, results.get("outputStream").contains("odpi_runtime_hive")); + }else{ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-i").addArgument("hive-init2.sql").addArgument("-e").addArgument("SHOW DATABASES").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("Could not create database odpi_runtime_hive.", 0, Integer.parseInt(results.get("exitValue"))); + Assert.assertEquals("Could not create database odpi_runtime_hive using the init -i option.", true, results.get("outputStream").contains("odpi_runtime_hive")); + } + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("DROP DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + } + + @Test + public void database(){ + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("SHOW DATABASES").addArgument("--hiveconf").addArgument(db)); + if(!results.get("outputStream").contains("odpi_runtime_hive")){ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("CREATE DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + }else{ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("DROP DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("CREATE DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + } + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("--database").addArgument("odpi_runtime_hive_1234").addArgument("-e").addArgument("CREATE TABLE odpi ( MYID INT );").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("Non-existent database returned with wrong exit code: "+Integer.parseInt(results.get("exitValue")), 88, Integer.parseInt(results.get("exitValue"))); + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("--database").addArgument("odpi_runtime_hive").addArgument("-e").addArgument("CREATE TABLE odpi ( MYID INT );").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("Failed to create table using --database argument.", 0, Integer.parseInt(results.get("exitValue"))); + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("--database").addArgument("odpi_runtime_hive").addArgument("-e").addArgument("DESCRIBE odpi").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("Failed to get expected column after creating odpi table using --database argument.", true, results.get("outputStream").contains("myid")); + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("--database").addArgument("odpi_runtime_hive").addArgument("-e").addArgument("DROP TABLE odpi").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("Failed to create table using --database argument.", 0, Integer.parseInt(results.get("exitValue"))); + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("DROP DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + } + + @Test + public void hiveConf(){ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("--hiveconf").addArgument("hive.root.logger=INFO,console").addArgument("-e").addArgument("SHOW DATABASES").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("The --hiveconf option did not work in setting hive.root.logger=INFO,console.", true, results.get("outputStream").contains("INFO parse.ParseDriver: Parsing command: SHOW DATABASES")); + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-hiveconf").addArgument("hive.root.logger=INFO,console").addArgument("-e").addArgument("SHOW DATABASES").addArgument("--hiveconf").addArgument(db)); + Assert.assertEquals("The -hiveconf variant option did not work in setting hive.root.logger=INFO,console.", true, results.get("outputStream").contains("INFO parse.ParseDriver: Parsing command: SHOW DATABASES")); + } + + @Test + public void variableSubsitution() throws FileNotFoundException{ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("SHOW DATABASES").addArgument("--hiveconf").addArgument(db)); + if(!results.get("outputStream").contains("odpi_runtime_hive")){ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("CREATE DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + }else{ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("DROP DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("CREATE DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + } + try(PrintWriter out = new PrintWriter("hive-define.sql")){ out.println("show ${A};"); out.println("quit;"); } + results = HiveHelper.execCommand(new CommandLine("/bin/sh").addArgument("-c").addArgument("hive -d A=DATABASES --hiveconf '"+db+"' < hive-define.sql", false)); + Assert.assertEquals("The hive -d A=DATABASES option did not work.", 0, Integer.parseInt(results.get("exitValue"))); + Assert.assertEquals("The hive -d A=DATABASES option did not work.", true, results.get("outputStream").contains("odpi_runtime_hive")); + + results = HiveHelper.execCommand(new CommandLine("/bin/sh").addArgument("-c").addArgument("hive --define A=DATABASES --hiveconf '"+db+"' < hive-define.sql", false)); + Assert.assertEquals("The hive --define A=DATABASES option did not work.", 0, Integer.parseInt(results.get("exitValue"))); + Assert.assertEquals("The hive --define A=DATABASES option did not work.", true, results.get("outputStream").contains("odpi_runtime_hive")); + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("DROP DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + } + + @Test + public void hiveVar() throws FileNotFoundException{ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("SHOW DATABASES").addArgument("--hiveconf").addArgument(db)); + if(!results.get("outputStream").contains("odpi_runtime_hive")){ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("CREATE DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + }else{ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("DROP DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("CREATE DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + } + try(PrintWriter out = new PrintWriter("hive-var.sql")){ out.println("show ${A};"); out.println("quit;"); } + results = HiveHelper.execCommand(new CommandLine("/bin/sh").addArgument("-c").addArgument("hive --hivevar A=DATABASES --hiveconf '"+db+"' < hive-var.sql", false)); + Assert.assertEquals("The hive --hivevar A=DATABASES option did not work.", 0, Integer.parseInt(results.get("exitValue"))); + Assert.assertEquals("The hive --hivevar A=DATABASES option did not work.", true, results.get("outputStream").contains("odpi_runtime_hive")); + + try(PrintWriter out = new PrintWriter("hiveconf-var.sql")){ out.println("show ${hiveconf:A};"); out.println("quit;"); } + results = HiveHelper.execCommand(new CommandLine("/bin/sh").addArgument("-c").addArgument("hive --hiveconf A=DATABASES --hiveconf '"+db+"' < hiveconf-var.sql", false)); + Assert.assertEquals("The hive --hiveconf A=DATABASES option did not work.", 0, Integer.parseInt(results.get("exitValue"))); + Assert.assertEquals("The hive --hiveconf A=DATABASES option did not work.", true, results.get("outputStream").contains("odpi_runtime_hive")); + + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("DROP DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + } + + @AfterClass + public static void cleanup(){ + results = HiveHelper.execCommand(new CommandLine("hive").addArgument("-e").addArgument("DROP DATABASE odpi_runtime_hive").addArgument("--hiveconf").addArgument(db)); + results = HiveHelper.execCommand(new CommandLine("/bin/sh").addArgument("-c").addArgument("rm -rf hive-f*.sql", false)); + results = HiveHelper.execCommand(new CommandLine("/bin/sh").addArgument("-c").addArgument("rm -rf hive-init*.sql", false)); + results = HiveHelper.execCommand(new CommandLine("/bin/sh").addArgument("-c").addArgument("rm -rf hive-define.sql", false)); + results = HiveHelper.execCommand(new CommandLine("/bin/sh").addArgument("-c").addArgument("rm -rf hive-var.sql", false)); + results = HiveHelper.execCommand(new CommandLine("/bin/sh").addArgument("-c").addArgument("rm -rf hiveconf-var.sql", false)); + } + +}
