ODPI-185. Test [HIVE_HCATALOG]
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/94efd55c Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/94efd55c Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/94efd55c Branch: refs/heads/BIGTOP-1406 Commit: 94efd55c5c35334ae34a5733424f31245f601b4a Parents: 47c77bc Author: Roman Shaposhnik <[email protected]> Authored: Tue Nov 8 19:24:44 2016 -0800 Committer: Roman Shaposhnik <[email protected]> Committed: Fri Feb 24 12:03:52 2017 -0800 ---------------------------------------------------------------------- bigtop-tests/spec-tests/runtime/build.gradle | 13 +++++++++++++ .../java/org/odpi/specs/runtime/hive/HCatalogMR.java | 11 ----------- .../org/odpi/specs/runtime/hive/TestHCatalog.java | 15 +++++---------- 3 files changed, 18 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/94efd55c/bigtop-tests/spec-tests/runtime/build.gradle ---------------------------------------------------------------------- diff --git a/bigtop-tests/spec-tests/runtime/build.gradle b/bigtop-tests/spec-tests/runtime/build.gradle index f0166c9..a88a3b6 100644 --- a/bigtop-tests/spec-tests/runtime/build.gradle +++ b/bigtop-tests/spec-tests/runtime/build.gradle @@ -43,7 +43,20 @@ dependencies { if (System.env.HADOOP_CONF_DIR) testRuntime files(System.env.HADOOP_CONF_DIR) } +jar { + from { + (configurations.runtime).grep{it.toString() =~ /(hive|libfb303)-.*[jw]ar$/}.collect { + zipTree(it) + } + } + + exclude 'META-INF/*.RSA', 'META-INF/*.SF','META-INF/*.DSA' +} + test { // Change the default location where test data is picked up systemProperty 'test.resources.dir', "${buildDir}/resources/test/" + systemProperty 'odpi.test.hive.hcat.job.jar', jar.archivePath + systemProperty 'odpi.test.hive.hcat.core.jar', (configurations.runtime).find { it.toString() =~ /hive-hcatalog-core-.*jar$/ } } +test.dependsOn jar http://git-wip-us.apache.org/repos/asf/bigtop/blob/94efd55c/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java b/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java index a6ff375..7cb9bbe 100644 --- a/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java +++ b/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java @@ -72,17 +72,6 @@ public class HCatalogMR extends Configured implements Tool { HCatOutputFormat.setSchema(job, HCatSchemaUtils.getHCatSchema(outputSchemaStr)); job.setOutputFormatClass(HCatOutputFormat.class); - // TODO All four of these jars need to be in the distributed cache of the job for the job to - // succeed. I loaded them into a known location in HDFS to get them in the cache. There may - // be a way to load them from a file on the gateway machine. We could also put in a hdfs dfs - // -put operation into a gradle step as part of the build so that the jars are picked up from - // the distribution and put in a known location in HDFS from when they can be picked up in - // the distributed cache. - job.addCacheArchive(new URI("hdfs:/user/gates/hive-hcatalog-core-1.2.1.jar")); - job.addCacheArchive(new URI("hdfs:/user/gates/hive-metastore-1.2.1.jar")); - job.addCacheArchive(new URI("hdfs:/user/gates/hive-exec-1.2.1.jar")); - job.addCacheArchive(new URI("hdfs:/user/gates/libfb303-0.9.2.jar")); - return job.waitForCompletion(true) ? 0 : 1; http://git-wip-us.apache.org/repos/asf/bigtop/blob/94efd55c/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java b/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java index 87e3eb0..b51db02 100644 --- a/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java +++ b/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java @@ -51,6 +51,8 @@ import java.util.Random; public class TestHCatalog { + private static final String JOBJAR = "odpi.test.hive.hcat.job.jar"; + private static final String HCATCORE = "odpi.test.hive.hcat.core.jar"; private static final Log LOG = LogFactory.getLog(TestHCatalog.class.getName()); @@ -132,18 +134,11 @@ public class TestHCatalog { out.close(); Map<String, String> env = new HashMap<>(); - // TODO These need to be set from the environment rather than hard wired - env.put("HADOOP_HOME","/Users/gates/grid/odpi-testing/hadoop-2.7.3"); - env.put("HADOOP_CLASSPATH", "/Users/gates/grid/odpi-testing/apache-hive-1.2.1-bin/hcatalog/share/hcatalog/hive-hcatalog-core-1.2.1.jar"); - env.put("HIVE_HOME", "/Users/gates/grid/odpi-testing/apache-hive-1.2.1-bin"); - Map<String, String> results = HiveHelper.execCommand(new CommandLine("/Users/gates/grid/odpi-testing/apache-hive-1.2.1-bin/bin/hive") + env.put("HADOOP_CLASSPATH", System.getProperty(HCATCORE, "")); + Map<String, String> results = HiveHelper.execCommand(new CommandLine("hive") .addArgument("--service") .addArgument("jar") - // TODO This is the jar built by gradle, but I didn't know how to take the jar built in - // the build phase and reference it in the test phase. Perhaps a move operation could be - // put in the middle so the jar is moved to a known location that can be referenced here, - // or maybe gradle can pass in its working directory so that we can reference it from there. - .addArgument("/Users/gates/git/bigtop/runtime-1.2.0-SNAPSHOT.jar") + .addArgument(System.getProperty(JOBJAR)) .addArgument(HCatalogMR.class.getName()) .addArgument(inputTable) .addArgument(outputTable)
