Added TODOs to outline remaining work.
Project: http://git-wip-us.apache.org/repos/asf/bigtop/repo Commit: http://git-wip-us.apache.org/repos/asf/bigtop/commit/e5518cdb Tree: http://git-wip-us.apache.org/repos/asf/bigtop/tree/e5518cdb Diff: http://git-wip-us.apache.org/repos/asf/bigtop/diff/e5518cdb Branch: refs/heads/BIGTOP-2666 Commit: e5518cdb461165b0db045f05f81b436343f2f4a0 Parents: 1b4f286 Author: Alan Gates <[email protected]> Authored: Tue Nov 8 07:42:28 2016 -0800 Committer: Roman Shaposhnik <[email protected]> Committed: Fri Feb 24 11:51:24 2017 -0800 ---------------------------------------------------------------------- .../src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java | 6 ++++++ .../test/java/org/odpi/specs/runtime/hive/TestHCatalog.java | 7 ++++++- 2 files changed, 12 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/bigtop/blob/e5518cdb/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java b/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java index 6fcfe37..a6ff375 100644 --- a/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java +++ b/bigtop-tests/spec-tests/runtime/src/main/java/org/odpi/specs/runtime/hive/HCatalogMR.java @@ -72,6 +72,12 @@ public class HCatalogMR extends Configured implements Tool { HCatOutputFormat.setSchema(job, HCatSchemaUtils.getHCatSchema(outputSchemaStr)); job.setOutputFormatClass(HCatOutputFormat.class); + // TODO All four of these jars need to be in the distributed cache of the job for the job to + // succeed. I loaded them into a known location in HDFS to get them in the cache. There may + // be a way to load them from a file on the gateway machine. We could also put in a hdfs dfs + // -put operation into a gradle step as part of the build so that the jars are picked up from + // the distribution and put in a known location in HDFS from when they can be picked up in + // the distributed cache. job.addCacheArchive(new URI("hdfs:/user/gates/hive-hcatalog-core-1.2.1.jar")); job.addCacheArchive(new URI("hdfs:/user/gates/hive-metastore-1.2.1.jar")); job.addCacheArchive(new URI("hdfs:/user/gates/hive-exec-1.2.1.jar")); http://git-wip-us.apache.org/repos/asf/bigtop/blob/e5518cdb/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java ---------------------------------------------------------------------- diff --git a/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java b/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java index bb237d8..87e3eb0 100644 --- a/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java +++ b/bigtop-tests/spec-tests/runtime/src/test/java/org/odpi/specs/runtime/hive/TestHCatalog.java @@ -118,7 +118,7 @@ public class TestHCatalog { new HCatFieldSchema("word", HCatFieldSchema.Type.STRING, ""), new HCatFieldSchema("count", HCatFieldSchema.Type.INT, ""))); - // TODO Could I use HCatWriter hear and the reader to read it? + // LATER Could I use HCatWriter here and the reader to read it? // Write some stuff into a file in the location of the table table = client.getTable("default", inputTable); String inputFile = table.getSd().getLocation() + "/input"; @@ -132,12 +132,17 @@ public class TestHCatalog { out.close(); Map<String, String> env = new HashMap<>(); + // TODO These need to be set from the environment rather than hard wired env.put("HADOOP_HOME","/Users/gates/grid/odpi-testing/hadoop-2.7.3"); env.put("HADOOP_CLASSPATH", "/Users/gates/grid/odpi-testing/apache-hive-1.2.1-bin/hcatalog/share/hcatalog/hive-hcatalog-core-1.2.1.jar"); env.put("HIVE_HOME", "/Users/gates/grid/odpi-testing/apache-hive-1.2.1-bin"); Map<String, String> results = HiveHelper.execCommand(new CommandLine("/Users/gates/grid/odpi-testing/apache-hive-1.2.1-bin/bin/hive") .addArgument("--service") .addArgument("jar") + // TODO This is the jar built by gradle, but I didn't know how to take the jar built in + // the build phase and reference it in the test phase. Perhaps a move operation could be + // put in the middle so the jar is moved to a known location that can be referenced here, + // or maybe gradle can pass in its working directory so that we can reference it from there. .addArgument("/Users/gates/git/bigtop/runtime-1.2.0-SNAPSHOT.jar") .addArgument(HCatalogMR.class.getName()) .addArgument(inputTable)
