Repository: hive Updated Branches: refs/heads/branch-3 5ec8e356d -> ac5de3d45
HIVE-19467: Make storage format configurable for temp tables created using LLAP external client (Jason Dere, reviewed by Deepak Jaiswal) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ac5de3d4 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ac5de3d4 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ac5de3d4 Branch: refs/heads/branch-3 Commit: ac5de3d45ce33ed61a00f5a2f3637cf755d3ad82 Parents: 5ec8e35 Author: Jason Dere <[email protected]> Authored: Wed May 9 18:05:50 2018 -0700 Committer: Jason Dere <[email protected]> Committed: Mon Jun 4 15:30:56 2018 -0700 ---------------------------------------------------------------------- .../java/org/apache/hadoop/hive/conf/HiveConf.java | 3 +++ .../org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java | 13 +++++++++++++ .../hive/ql/udf/generic/GenericUDTFGetSplits.java | 15 ++++++++++++++- 3 files changed, 30 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/ac5de3d4/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 8347f7f..6939dd0 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -4149,6 +4149,9 @@ public class HiveConf extends Configuration { LLAP_DAEMON_OUTPUT_SERVICE_MAX_PENDING_WRITES("hive.llap.daemon.output.service.max.pending.writes", 8, "Maximum number of queued writes allowed per connection when sending data\n" + " via the LLAP output service to external clients."), + LLAP_EXTERNAL_SPLITS_TEMP_TABLE_STORAGE_FORMAT("hive.llap.external.splits.temp.table.storage.format", + "orc", new StringSet("default", "text", "orc"), + "Storage format for temp tables created using LLAP external client"), LLAP_ENABLE_GRACE_JOIN_IN_LLAP("hive.llap.enable.grace.join.in.llap", false, "Override if grace join should be allowed to run in llap."), http://git-wip-us.apache.org/repos/asf/hive/blob/ac5de3d4/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java index 11017f6..7a891ef 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/BaseJdbcWithMiniLlap.java @@ -448,6 +448,19 @@ public abstract class BaseJdbcWithMiniLlap { assertArrayEquals("X'01FF'".getBytes("UTF-8"), (byte[]) rowValues[22]); } + + @Test(timeout = 60000) + public void testComplexQuery() throws Exception { + createTestTable("testtab1"); + + RowCollector rowCollector = new RowCollector(); + String query = "select value, count(*) from testtab1 where under_col=0 group by value"; + int rowCount = processQuery(query, 1, rowCollector); + assertEquals(1, rowCount); + + assertArrayEquals(new String[] {"val_0", "3"}, rowCollector.rows.get(0)); + } + private interface RowProcessor { void process(Row row); } http://git-wip-us.apache.org/repos/asf/hive/blob/ac5de3d4/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java index 7dbde7a..20d0961 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDTFGetSplits.java @@ -293,7 +293,8 @@ public class GenericUDTFGetSplits extends GenericUDTF { String tableName = "table_"+UUID.randomUUID().toString().replaceAll("[^A-Za-z0-9 ]", ""); - String ctas = "create temporary table " + tableName + " as " + query; + String storageFormatString = getTempTableStorageFormatString(conf); + String ctas = "create temporary table " + tableName + " " + storageFormatString + " as " + query; LOG.info("Materializing the query for LLAPIF; CTAS: " + ctas); driver.releaseResources(); HiveConf.setVar(conf, ConfVars.HIVE_EXECUTION_MODE, originalMode); @@ -674,6 +675,18 @@ public class GenericUDTFGetSplits extends GenericUDTF { return Schema; } + private String getTempTableStorageFormatString(HiveConf conf) { + String formatString = ""; + String storageFormatOption = + conf.getVar(HiveConf.ConfVars.LLAP_EXTERNAL_SPLITS_TEMP_TABLE_STORAGE_FORMAT).toLowerCase(); + if (storageFormatOption.equals("text")) { + formatString = "stored as textfile"; + } else if (storageFormatOption.equals("orc")) { + formatString = "stored as orc"; + } + return formatString; + } + @Override public void close() throws HiveException { }
