HAWQ-1228. Hardcoded comma as delimiter.
Project: http://git-wip-us.apache.org/repos/asf/incubator-hawq/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-hawq/commit/8fe27032 Tree: http://git-wip-us.apache.org/repos/asf/incubator-hawq/tree/8fe27032 Diff: http://git-wip-us.apache.org/repos/asf/incubator-hawq/diff/8fe27032 Branch: refs/heads/HAWQ-1228 Commit: 8fe27032f1faf786544a5109e854d319f6e6a1c4 Parents: 0574e75 Author: Oleksandr Diachenko <[email protected]> Authored: Fri Jan 6 18:39:32 2017 -0800 Committer: Oleksandr Diachenko <[email protected]> Committed: Fri Jan 6 18:39:32 2017 -0800 ---------------------------------------------------------------------- .../org/apache/hawq/pxf/api/OutputFormat.java | 2 +- .../plugins/hive/HiveColumnarSerdeResolver.java | 2 +- .../pxf/plugins/hive/HiveMetadataFetcher.java | 2 +- .../pxf/plugins/hive/HiveORCSerdeResolver.java | 22 -------------------- .../hawq/pxf/plugins/hive/HiveResolver.java | 3 --- .../plugins/hive/HiveStringPassResolver.java | 2 +- src/backend/catalog/external/externalmd.c | 2 +- src/bin/gpfusion/gpbridgeapi.c | 4 ++-- src/include/access/hd_work_mgr.h | 2 ++ 9 files changed, 9 insertions(+), 32 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java index 82a747f..230f9ff 100644 --- a/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java +++ b/pxf/pxf-api/src/main/java/org/apache/hawq/pxf/api/OutputFormat.java @@ -23,4 +23,4 @@ package org.apache.hawq.pxf.api; /** * PXF supported output formats: {@link #TEXT} and {@link #BINARY} */ -public enum OutputFormat {TEXT, BINARY, UNKNOWN} +public enum OutputFormat {TEXT, BINARY} http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java index 2bf39ff..8fd37e0 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveColumnarSerdeResolver.java @@ -241,6 +241,6 @@ public class HiveColumnarSerdeResolver extends HiveResolver { @Override void parseDelimiterChar(InputData input) { - delimiter = 1; + delimiter = 44; //, } } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java index 3d04bc1..76e8e18 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveMetadataFetcher.java @@ -162,7 +162,7 @@ public class HiveMetadataFetcher extends MetadataFetcher { } private OutputFormat getOutputFormat(String inputFormat) { - OutputFormat outputFormat = OutputFormat.UNKNOWN; + OutputFormat outputFormat = null; try { InputFormat<?, ?> fformat = HiveDataFragmenter.makeInputFormat(inputFormat, jobConf); String profile = ProfileFactory.get(fformat); http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java index 381c407..0c7a417 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveORCSerdeResolver.java @@ -44,7 +44,6 @@ import java.util.*; */ public class HiveORCSerdeResolver extends HiveResolver { private static final Log LOG = LogFactory.getLog(HiveORCSerdeResolver.class); - private OrcSerde deserializer; private HiveUtilities.PXF_HIVE_SERDES serdeType; public HiveORCSerdeResolver(InputData input) throws Exception { @@ -63,27 +62,6 @@ public class HiveORCSerdeResolver extends HiveResolver { : input.getUserProperty("MAPKEY_DELIM"); } - /** - * getFields returns a singleton list of OneField item. - * OneField item contains two fields: an integer representing the VARCHAR type and a Java - * Object representing the field value. - */ - //TODO: It's the same as in parent class - @Override - public List<OneField> getFields(OneRow onerow) throws Exception { - - Object tuple = deserializer.deserialize((Writable) onerow.getData()); - // Each Hive record is a Struct - StructObjectInspector soi = (StructObjectInspector) deserializer.getObjectInspector(); - List<OneField> record = traverseStruct(tuple, soi, false); - - //Add partition fields if any - record.addAll(getPartitionFields()); - - return record; - - } - /* * Get and init the deserializer for the records of this Hive data fragment. * Suppress Warnings added because deserializer.initialize is an abstract function that is deprecated http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java index 55d7205..b0dc2fb 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveResolver.java @@ -75,12 +75,9 @@ public class HiveResolver extends Plugin implements ReadResolver { protected static final String COLLECTION_DELIM = ","; protected String collectionDelim; protected String mapkeyDelim; - //private SerDe deserializer; protected SerDe deserializer; private List<OneField> partitionFields; - //private String serdeClassName; protected String serdeClassName; - //private String propsString; protected String propsString; String partitionKeys; protected char delimiter; http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java ---------------------------------------------------------------------- diff --git a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java index c7cfb36..8c91d47 100644 --- a/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java +++ b/pxf/pxf-hive/src/main/java/org/apache/hawq/pxf/plugins/hive/HiveStringPassResolver.java @@ -93,6 +93,6 @@ public class HiveStringPassResolver extends HiveResolver { } void parseDelimiterChar(InputData input) { - this.delimiter = 1; + delimiter = 44; //, } } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/src/backend/catalog/external/externalmd.c ---------------------------------------------------------------------- diff --git a/src/backend/catalog/external/externalmd.c b/src/backend/catalog/external/externalmd.c index ccdbdd6..c6cfb69 100644 --- a/src/backend/catalog/external/externalmd.c +++ b/src/backend/catalog/external/externalmd.c @@ -659,7 +659,7 @@ static Datum GetFormatOptionsForProfile(const List *outputFormats) initStringInfo(&formatStr); if (list_length(outputFormats) == 1 && strcmp(lfirst(list_head(outputFormats)),"TEXT") == 0) { - appendStringInfo(&formatStr, "delimiter '\x01' null '\N' escape '\'"); + appendStringInfo(&formatStr, "delimiter '\x2C' null '\\N' escape '\\'"); } else { appendStringInfo(&formatStr, "formatter 'pxfwritable_import'"); } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/src/bin/gpfusion/gpbridgeapi.c ---------------------------------------------------------------------- diff --git a/src/bin/gpfusion/gpbridgeapi.c b/src/bin/gpfusion/gpbridgeapi.c index c5c217c..358c858 100644 --- a/src/bin/gpfusion/gpbridgeapi.c +++ b/src/bin/gpfusion/gpbridgeapi.c @@ -564,7 +564,7 @@ void free_token_resources(PxfInputData *inputData) static void assign_optimal_supported_profile(char *profile, char *fmttype, char **supportedProfile, char **supportedFormat) { - if (fmttype_is_text(*fmttype) && ((strcmp(profile, "HiveText") == 0) || (strcmp(profile, "HiveRc") == 0))) + if (fmttype_is_text(*fmttype) && ((strcmp(profile, HiveTextProfileName) == 0) || (strcmp(profile, HiveRCProfileName) == 0))) { *supportedFormat = "TEXT"; *supportedProfile = profile; @@ -575,6 +575,6 @@ static void assign_optimal_supported_profile(char *profile, char *fmttype, char } else { *supportedFormat = "GPDBWritable"; - *supportedProfile = "Hive"; + *supportedProfile = HiveProfileName; } } http://git-wip-us.apache.org/repos/asf/incubator-hawq/blob/8fe27032/src/include/access/hd_work_mgr.h ---------------------------------------------------------------------- diff --git a/src/include/access/hd_work_mgr.h b/src/include/access/hd_work_mgr.h index cab8ca7..ea4c6ef 100644 --- a/src/include/access/hd_work_mgr.h +++ b/src/include/access/hd_work_mgr.h @@ -48,5 +48,7 @@ PxfFragmentStatsElem *get_pxf_fragments_statistics(char *uri, Relation rel); List *get_pxf_item_metadata(char *profile, char *pattern, Oid dboid); #define HiveProfileName "Hive" +#define HiveTextProfileName "HiveText" +#define HiveRCProfileName "HiveRC" #endif /* HDWORKMGR_H */
