Repository: incubator-hawq Updated Branches: refs/heads/master 01373b943 -> 52333bb00
HAWQ-1196. Added partitons support to HiveOrc profile. Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/52333bb0 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/52333bb0 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/52333bb0 Branch: refs/heads/master Commit: 52333bb0099b34c8c354f78c10a235dc377560cf Parents: 01373b9 Author: Oleksandr Diachenko <[email protected]> Authored: Wed Dec 7 15:16:11 2016 -0800 Committer: Oleksandr Diachenko <[email protected]> Committed: Wed Dec 7 15:16:11 2016 -0800 ---------------------------------------------------------------------- .../pxf/plugins/hive/HiveColumnarSerdeResolver.java | 5 ++--- .../hawq/pxf/plugins/hive/HiveORCSerdeResolver.java | 13 ++++--------- .../apache/hawq/pxf/plugins/hive/HiveResolver.java | 16 +++++++++++++--- 3 files changed, 19 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/52333bb0/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java index 987fc0f..362ac0d 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java @@ -61,7 +61,6 @@ public class HiveColumnarSerdeResolver extends HiveResolver { private boolean firstColumn; private StringBuilder builder; private StringBuilder parts; - private int numberOfPartitions; private HiveInputFormatFragmenter.PXF_HIVE_SERDES serdeType; public HiveColumnarSerdeResolver(InputData input) throws Exception { @@ -88,7 +87,7 @@ public class HiveColumnarSerdeResolver extends HiveResolver { @Override void initPartitionFields() { - numberOfPartitions = initPartitionFields(parts); + initPartitionFields(parts); } /** @@ -118,7 +117,7 @@ public class HiveColumnarSerdeResolver extends HiveResolver { @Override void initSerde(InputData input) throws Exception { Properties serdeProperties = new Properties(); - int numberOfDataColumns = input.getColumns() - numberOfPartitions; + int numberOfDataColumns = input.getColumns() - getNumberOfPartitions(); LOG.debug("Serde number of columns is " + numberOfDataColumns); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/52333bb0/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java index 9e32633..c789402 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java @@ -44,8 +44,6 @@ import java.util.*; public class HiveORCSerdeResolver extends HiveResolver { private static final Log LOG = LogFactory.getLog(HiveORCSerdeResolver.class); private OrcSerde deserializer; - private StringBuilder parts; - private int numberOfPartitions; private HiveInputFormatFragmenter.PXF_HIVE_SERDES serdeType; public HiveORCSerdeResolver(InputData input) throws Exception { @@ -70,12 +68,6 @@ public class HiveORCSerdeResolver extends HiveResolver { : input.getUserProperty("MAPKEY_DELIM"); } - @Override - void initPartitionFields() { - parts = new StringBuilder(); - numberOfPartitions = initPartitionFields(parts); - } - /** * getFields returns a singleton list of OneField item. * OneField item contains two fields: an integer representing the VARCHAR type and a Java @@ -89,6 +81,9 @@ public class HiveORCSerdeResolver extends HiveResolver { StructObjectInspector soi = (StructObjectInspector) deserializer.getObjectInspector(); List<OneField> record = traverseStruct(tuple, soi, false); + //Add partition fields if any + record.addAll(getPartitionFields()); + return record; } @@ -102,7 +97,7 @@ public class HiveORCSerdeResolver extends HiveResolver { @Override void initSerde(InputData input) throws Exception { Properties serdeProperties = new Properties(); - int numberOfDataColumns = input.getColumns() - numberOfPartitions; + int numberOfDataColumns = input.getColumns() - getNumberOfPartitions(); LOG.debug("Serde number of columns is " + numberOfDataColumns); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/52333bb0/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java index 93d5bdb..d5e6e96 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java @@ -83,6 +83,7 @@ public class HiveResolver extends Plugin implements ReadResolver { String nullChar = "\\N"; private Configuration conf; private String hiveDefaultPartName; + private int numberOfPartitions; /** * Constructs the HiveResolver by parsing the userdata in the input and @@ -122,6 +123,14 @@ public class HiveResolver extends Plugin implements ReadResolver { return record; } + public List<OneField> getPartitionFields() { + return partitionFields; + } + + public int getNumberOfPartitions() { + return numberOfPartitions; + } + /* Parses user data string (arrived from fragmenter). */ void parseUserData(InputData input) throws Exception { final int EXPECTED_NUM_OF_TOKS = 5; @@ -254,6 +263,7 @@ public class HiveResolver extends Plugin implements ReadResolver { "Unsupported partition type: " + type); } addOneFieldToRecord(partitionFields, convertedType, convertedValue); + numberOfPartitions = partitionFields.size(); } } @@ -261,9 +271,9 @@ public class HiveResolver extends Plugin implements ReadResolver { * The partition fields are initialized one time based on userData provided * by the fragmenter. */ - int initPartitionFields(StringBuilder parts) { + void initPartitionFields(StringBuilder parts) { if (partitionKeys.equals(HiveDataFragmenter.HIVE_NO_PART_TBL)) { - return 0; + return; } String[] partitionLevels = partitionKeys.split(HiveDataFragmenter.HIVE_PARTITIONS_DELIM); for (String partLevel : partitionLevels) { @@ -320,7 +330,7 @@ public class HiveResolver extends Plugin implements ReadResolver { } } } - return partitionLevels.length; + this.numberOfPartitions = partitionLevels.length; } /**
