[
https://issues.apache.org/jira/browse/PARQUET-1133?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Julien Le Dem resolved PARQUET-1133.
------------------------------------
Resolution: Fixed
Fix Version/s: 1.9.0
Issue resolved by pull request 422
[https://github.com/apache/parquet-mr/pull/422]
> INT96 types and Maps without OriginalType cause exceptions in
> PigSchemaConverter
> --------------------------------------------------------------------------------
>
> Key: PARQUET-1133
> URL: https://issues.apache.org/jira/browse/PARQUET-1133
> Project: Parquet
> Issue Type: Bug
> Components: parquet-pig
> Affects Versions: 1.9.1
> Reporter: Addisu Feyissa
> Assignee: Addisu Feyissa
> Fix For: 1.9.0
>
>
> Trying to load parquet files in Pig, that have the following causes an
> exception and parsing to fail:
> * INT96 fields, for example:
> {noformat}
> message spark_schema {
> optional int96 datetime;
> }
> {noformat}
> The Exception thrown is:
> {noformat}
> Failed to parse: can't convert optional int96 myInt96
> at
> org.apache.pig.parser.QueryParserDriver.parse(QueryParserDriver.java:201)
> at org.apache.pig.PigServer$Graph.validateQuery(PigServer.java:1791)
> at org.apache.pig.PigServer$Graph.registerQuery(PigServer.java:1764)
> at org.apache.pig.PigServer.registerQuery(PigServer.java:707)
> at
> org.apache.pig.tools.grunt.GruntParser.processPig(GruntParser.java:1075)
> at
> org.apache.pig.tools.pigscript.parser.PigScriptParser.parse(PigScriptParser.java:505)
> at
> org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:231)
> at
> org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:206)
> at org.apache.pig.tools.grunt.Grunt.run(Grunt.java:66)
> at org.apache.pig.Main.run(Main.java:564)
> at org.apache.pig.Main.main(Main.java:176)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.hadoop.util.RunJar.run(RunJar.java:234)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:148)
> Caused by: org.apache.parquet.pig.SchemaConversionException: can't convert
> optional int96 myInt96
> at
> org.apache.parquet.pig.PigSchemaConverter.convertFields(PigSchemaConverter.java:202)
> at
> org.apache.parquet.pig.PigSchemaConverter.convert(PigSchemaConverter.java:178)
> at
> org.apache.parquet.pig.TupleReadSupport.getPigSchemaFromMultipleFiles(TupleReadSupport.java:95)
> at
> org.apache.parquet.pig.ParquetLoader.initSchema(ParquetLoader.java:300)
> at org.apache.parquet.pig.ParquetLoader.setInput(ParquetLoader.java:183)
> at
> org.apache.parquet.pig.ParquetLoader.getSchema(ParquetLoader.java:285)
> at
> org.apache.pig.newplan.logical.relational.LOLoad.getSchemaFromMetaData(LOLoad.java:175)
> at
> org.apache.pig.newplan.logical.relational.LOLoad.<init>(LOLoad.java:89)
> at
> org.apache.pig.parser.LogicalPlanBuilder.buildLoadOp(LogicalPlanBuilder.java:901)
> at
> org.apache.pig.parser.LogicalPlanGenerator.load_clause(LogicalPlanGenerator.java:3568)
> at
> org.apache.pig.parser.LogicalPlanGenerator.op_clause(LogicalPlanGenerator.java:1625)
> at
> org.apache.pig.parser.LogicalPlanGenerator.general_statement(LogicalPlanGenerator.java:1102)
> at
> org.apache.pig.parser.LogicalPlanGenerator.statement(LogicalPlanGenerator.java:560)
> at
> org.apache.pig.parser.LogicalPlanGenerator.query(LogicalPlanGenerator.java:421)
> at
> org.apache.pig.parser.QueryParserDriver.parse(QueryParserDriver.java:191)
> ... 16 more
> Caused by: org.apache.pig.impl.logicalLayer.FrontendException: ERROR 0: NYI
> at
> org.apache.parquet.pig.PigSchemaConverter$1.convertINT96(PigSchemaConverter.java:242)
> at
> org.apache.parquet.pig.PigSchemaConverter$1.convertINT96(PigSchemaConverter.java:214)
> at
> org.apache.parquet.schema.PrimitiveType$PrimitiveTypeName$7.convert(PrimitiveType.java:223)
> at
> org.apache.parquet.pig.PigSchemaConverter.getSimpleFieldSchema(PigSchemaConverter.java:213)
> at
> org.apache.parquet.pig.PigSchemaConverter.getFieldSchema(PigSchemaConverter.java:320)
> at
> org.apache.parquet.pig.PigSchemaConverter.convertFields(PigSchemaConverter.java:193)
> ... 30 more
> {noformat}
> * Map Types without OriginalType, for example:
> {noformat}
> message spark_schema {
> optional binary a;
> optional group b (MAP) {
> repeated group map {
> required binary key;
> optional group value {
> optional fixed_len_byte_array(5) c;
> optional fixed_len_byte_array(7) d;
> }
> }
> }
> }
> {noformat}
> The Exception thrown is:
> {noformat}
> ERROR 1200: null
> Failed to parse: null
> at
> org.apache.pig.parser.QueryParserDriver.parse(QueryParserDriver.java:201)
> at org.apache.pig.PigServer$Graph.validateQuery(PigServer.java:1791)
> at org.apache.pig.PigServer$Graph.registerQuery(PigServer.java:1764)
> at org.apache.pig.PigServer.registerQuery(PigServer.java:707)
> at
> org.apache.pig.tools.grunt.GruntParser.processPig(GruntParser.java:1075)
> at
> org.apache.pig.tools.pigscript.parser.PigScriptParser.parse(PigScriptParser.java:505)
> at
> org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:231)
> at
> org.apache.pig.tools.grunt.GruntParser.parseStopOnError(GruntParser.java:206)
> at org.apache.pig.tools.grunt.Grunt.run(Grunt.java:66)
> at org.apache.pig.Main.run(Main.java:564)
> at org.apache.pig.Main.main(Main.java:176)
> at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
> at
> sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
> at
> sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
> at java.lang.reflect.Method.invoke(Method.java:498)
> at org.apache.hadoop.util.RunJar.run(RunJar.java:234)
> at org.apache.hadoop.util.RunJar.main(RunJar.java:148)
> Caused by: java.lang.NullPointerException
> at
> org.apache.parquet.pig.PigSchemaConverter.getComplexFieldSchema(PigSchemaConverter.java:281)
> at
> org.apache.parquet.pig.PigSchemaConverter.getFieldSchema(PigSchemaConverter.java:322)
> at
> org.apache.parquet.pig.PigSchemaConverter.convertFields(PigSchemaConverter.java:193)
> at
> org.apache.parquet.pig.PigSchemaConverter.convert(PigSchemaConverter.java:178)
> at
> org.apache.parquet.pig.TupleReadSupport.getPigSchemaFromMultipleFiles(TupleReadSupport.java:95)
> at
> org.apache.parquet.pig.ParquetLoader.initSchema(ParquetLoader.java:300)
> at org.apache.parquet.pig.ParquetLoader.setInput(ParquetLoader.java:183)
> at
> org.apache.parquet.pig.ParquetLoader.getSchema(ParquetLoader.java:285)
> at
> org.apache.pig.newplan.logical.relational.LOLoad.getSchemaFromMetaData(LOLoad.java:175)
> at
> org.apache.pig.newplan.logical.relational.LOLoad.<init>(LOLoad.java:89)
> at
> org.apache.pig.parser.LogicalPlanBuilder.buildLoadOp(LogicalPlanBuilder.java:901)
> at
> org.apache.pig.parser.LogicalPlanGenerator.load_clause(LogicalPlanGenerator.java:3568)
> at
> org.apache.pig.parser.LogicalPlanGenerator.op_clause(LogicalPlanGenerator.java:1625)
> at
> org.apache.pig.parser.LogicalPlanGenerator.general_statement(LogicalPlanGenerator.java:1102)
> at
> org.apache.pig.parser.LogicalPlanGenerator.statement(LogicalPlanGenerator.java:560)
> at
> org.apache.pig.parser.LogicalPlanGenerator.query(LogicalPlanGenerator.java:421)
> at
> org.apache.pig.parser.QueryParserDriver.parse(QueryParserDriver.java:191)
> ... 16 more
> {noformat}
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
