Repository: incubator-blur Updated Branches: refs/heads/master b930ea235 -> 774d143c6
Attempting to make unit tests more consistent by adding an option to make hive output format block waiting for commit. Project: http://git-wip-us.apache.org/repos/asf/incubator-blur/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-blur/commit/774d143c Tree: http://git-wip-us.apache.org/repos/asf/incubator-blur/tree/774d143c Diff: http://git-wip-us.apache.org/repos/asf/incubator-blur/diff/774d143c Branch: refs/heads/master Commit: 774d143c62ef8131d0d882484a97ce828836c8d2 Parents: b930ea2 Author: Aaron McCurry <[email protected]> Authored: Mon Feb 9 08:02:09 2015 -0500 Committer: Aaron McCurry <[email protected]> Committed: Mon Feb 9 08:02:09 2015 -0500 ---------------------------------------------------------------------- .../java/org/apache/blur/hive/BlurHiveOutputCommitter.java | 8 ++++++-- blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java | 4 +++- .../src/test/java/org/apache/blur/hive/BlurSerDeTest.java | 4 +++- 3 files changed, 12 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/774d143c/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputCommitter.java ---------------------------------------------------------------------- diff --git a/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputCommitter.java b/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputCommitter.java index df182b3..bfeaaec 100644 --- a/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputCommitter.java +++ b/blur-hive/src/main/java/org/apache/blur/hive/BlurHiveOutputCommitter.java @@ -16,6 +16,9 @@ */ package org.apache.blur.hive; +import static org.apache.blur.hive.BlurSerDe.BLUR_BLOCKING_APPLY; +import static org.apache.blur.hive.BlurSerDe.BLUR_CONTROLLER_CONNECTION_STR; + import java.io.IOException; import org.apache.blur.thirdparty.thrift_0_9_0.TException; @@ -68,11 +71,12 @@ public class BlurHiveOutputCommitter extends OutputCommitter { private void finishBulkJob(JobContext context, boolean apply) throws IOException { Configuration configuration = context.getConfiguration(); - String connectionStr = configuration.get(BlurSerDe.BLUR_CONTROLLER_CONNECTION_STR); + String connectionStr = configuration.get(BLUR_CONTROLLER_CONNECTION_STR); + boolean blocking = configuration.getBoolean(BLUR_BLOCKING_APPLY, false); Iface client = BlurClient.getClient(connectionStr); String bulkId = BlurHiveOutputFormat.getBulkId(configuration); try { - client.bulkMutateFinish(bulkId, apply, false); + client.bulkMutateFinish(bulkId, apply, blocking); } catch (BlurException e) { throw new IOException(e); } catch (TException e) { http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/774d143c/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java ---------------------------------------------------------------------- diff --git a/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java b/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java index 055c6f4..150ad20 100644 --- a/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java +++ b/blur-hive/src/main/java/org/apache/blur/hive/BlurSerDe.java @@ -43,10 +43,12 @@ import org.apache.hadoop.io.Writable; public class BlurSerDe extends AbstractSerDe { - public static final String BLUR_CONTROLLER_CONNECTION_STR = "BLUR_CONTROLLER_CONNECTION_STR"; + public static final String BLUR_BLOCKING_APPLY = "blur.blocking.apply"; + public static final String BLUR_CONTROLLER_CONNECTION_STR = "blur.controller.connection.str"; public static final String FAMILY = "blur.family"; public static final String TABLE = "blur.table"; public static final String ZK = BlurConstants.BLUR_ZOOKEEPER_CONNECTION; + private String _family; private Map<String, ColumnDefinition> _schema; private ObjectInspector _objectInspector; http://git-wip-us.apache.org/repos/asf/incubator-blur/blob/774d143c/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java ---------------------------------------------------------------------- diff --git a/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java b/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java index b504ffe..8f37db3 100644 --- a/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java +++ b/blur-hive/src/test/java/org/apache/blur/hive/BlurSerDeTest.java @@ -256,7 +256,8 @@ public class BlurSerDeTest { } @Test - public void test2() throws SQLException, ClassNotFoundException, IOException, BlurException, TException, InterruptedException { + public void test2() throws SQLException, ClassNotFoundException, IOException, BlurException, TException, + InterruptedException { miniCluster.startMrMiniCluster(); Configuration configuration = miniCluster.getMRConfiguration(); HiveConf hiveConf = new HiveConf(configuration, getClass()); @@ -290,6 +291,7 @@ public class BlurSerDeTest { generateData(tableDir, totalRecords); run(connection, "select * from loadtable"); + run(connection, "set " + BlurSerDe.BLUR_BLOCKING_APPLY + " = true"); run(connection, "insert into table testtable select * from loadtable"); miniCluster.stopMrMiniCluster(); connection.close();
