Repository: hive Updated Branches: refs/heads/master de260b45d -> d4c1fdcf2
HIVE-13251: hive can't read the decimal in AVRO file generated from previous version (Reviewed by Szehon Ho) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d4c1fdcf Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d4c1fdcf Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d4c1fdcf Branch: refs/heads/master Commit: d4c1fdcf26ad8a12d77841def6afead86377b1ac Parents: de260b4 Author: Aihua Xu <aihu...@apache.org> Authored: Thu Mar 10 10:29:40 2016 -0500 Committer: Aihua Xu <aihu...@apache.org> Committed: Mon Mar 14 11:07:24 2016 -0400 ---------------------------------------------------------------------- data/files/dec_old.avro | Bin 0 -> 331 bytes .../test/queries/clientnegative/avro_decimal.q | 17 ++++++ .../queries/clientpositive/avro_decimal_old.q | 14 +++++ .../results/clientnegative/avro_decimal.q.out | 22 +++++++ .../clientpositive/avro_decimal_old.q.out | 60 +++++++++++++++++++ .../hive/serde2/avro/AvroDeserializer.java | 2 +- 6 files changed, 114 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/d4c1fdcf/data/files/dec_old.avro ---------------------------------------------------------------------- diff --git a/data/files/dec_old.avro b/data/files/dec_old.avro new file mode 100644 index 0000000..bf87763 Binary files /dev/null and b/data/files/dec_old.avro differ http://git-wip-us.apache.org/repos/asf/hive/blob/d4c1fdcf/ql/src/test/queries/clientnegative/avro_decimal.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientnegative/avro_decimal.q b/ql/src/test/queries/clientnegative/avro_decimal.q new file mode 100644 index 0000000..538e687 --- /dev/null +++ b/ql/src/test/queries/clientnegative/avro_decimal.q @@ -0,0 +1,17 @@ +DROP TABLE IF EXISTS avro_dec; + +CREATE TABLE `avro_dec`( + `name` string COMMENT 'from deserializer', + `value` decimal(5,2) COMMENT 'from deserializer') +COMMENT 'just drop the schema right into the HQL' +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +TBLPROPERTIES ( + 'numFiles'='1', + 'avro.schema.literal'='{\"namespace\":\"com.howdy\",\"name\":\"some_schema\",\"type\":\"record\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"value\",\"type\":{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":"5",\"scale\":"2"}}]}' +); + http://git-wip-us.apache.org/repos/asf/hive/blob/d4c1fdcf/ql/src/test/queries/clientpositive/avro_decimal_old.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/avro_decimal_old.q b/ql/src/test/queries/clientpositive/avro_decimal_old.q new file mode 100644 index 0000000..9610c47 --- /dev/null +++ b/ql/src/test/queries/clientpositive/avro_decimal_old.q @@ -0,0 +1,14 @@ +DROP TABLE IF EXISTS avro_dec_old; + +CREATE TABLE `avro_dec_old`( + `name` string COMMENT 'from deserializer', + `value` decimal(4,1) COMMENT 'from deserializer') +STORED AS AVRO; + +DESC avro_dec_old; + +LOAD DATA LOCAL INPATH '../../data/files/dec_old.avro' into TABLE avro_dec_old; + +select value from avro_dec_old; + +DROP TABLE avro_dec_old; http://git-wip-us.apache.org/repos/asf/hive/blob/d4c1fdcf/ql/src/test/results/clientnegative/avro_decimal.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientnegative/avro_decimal.q.out b/ql/src/test/results/clientnegative/avro_decimal.q.out new file mode 100644 index 0000000..9d00d6e --- /dev/null +++ b/ql/src/test/results/clientnegative/avro_decimal.q.out @@ -0,0 +1,22 @@ +PREHOOK: query: DROP TABLE IF EXISTS avro_dec +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS avro_dec +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE `avro_dec`( + `name` string COMMENT 'from deserializer', + `value` decimal(5,2) COMMENT 'from deserializer') +COMMENT 'just drop the schema right into the HQL' +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.avro.AvroSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat' +TBLPROPERTIES ( + 'numFiles'='1', + 'avro.schema.literal'='{\"namespace\":\"com.howdy\",\"name\":\"some_schema\",\"type\":\"record\",\"fields\":[{\"name\":\"name\",\"type\":\"string\"},{\"name\":\"value\",\"type\":{\"type\":\"bytes\",\"logicalType\":\"decimal\",\"precision\":"5",\"scale\":"2"}}]}' +) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_dec +FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. java.lang.RuntimeException: MetaException(message:org.apache.hadoop.hive.serde2.avro.AvroSerdeException Invalid precision or scale for decimal type) http://git-wip-us.apache.org/repos/asf/hive/blob/d4c1fdcf/ql/src/test/results/clientpositive/avro_decimal_old.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/avro_decimal_old.q.out b/ql/src/test/results/clientpositive/avro_decimal_old.q.out new file mode 100644 index 0000000..22efe39 --- /dev/null +++ b/ql/src/test/results/clientpositive/avro_decimal_old.q.out @@ -0,0 +1,60 @@ +PREHOOK: query: DROP TABLE IF EXISTS avro_dec_old +PREHOOK: type: DROPTABLE +POSTHOOK: query: DROP TABLE IF EXISTS avro_dec_old +POSTHOOK: type: DROPTABLE +PREHOOK: query: CREATE TABLE `avro_dec_old`( + `name` string COMMENT 'from deserializer', + `value` decimal(4,1) COMMENT 'from deserializer') +STORED AS AVRO +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@avro_dec_old +POSTHOOK: query: CREATE TABLE `avro_dec_old`( + `name` string COMMENT 'from deserializer', + `value` decimal(4,1) COMMENT 'from deserializer') +STORED AS AVRO +POSTHOOK: type: CREATETABLE +POSTHOOK: Output: database:default +POSTHOOK: Output: default@avro_dec_old +PREHOOK: query: DESC avro_dec_old +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@avro_dec_old +POSTHOOK: query: DESC avro_dec_old +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@avro_dec_old +name string from deserializer +value decimal(4,1) from deserializer +PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dec_old.avro' into TABLE avro_dec_old +PREHOOK: type: LOAD +#### A masked pattern was here #### +PREHOOK: Output: default@avro_dec_old +POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/dec_old.avro' into TABLE avro_dec_old +POSTHOOK: type: LOAD +#### A masked pattern was here #### +POSTHOOK: Output: default@avro_dec_old +PREHOOK: query: select value from avro_dec_old +PREHOOK: type: QUERY +PREHOOK: Input: default@avro_dec_old +#### A masked pattern was here #### +POSTHOOK: query: select value from avro_dec_old +POSTHOOK: type: QUERY +POSTHOOK: Input: default@avro_dec_old +#### A masked pattern was here #### +234.8 +77.3 +55.7 +4.3 +6.0 +12.3 +33.3 +19.0 +3.2 +79.9 +PREHOOK: query: DROP TABLE avro_dec_old +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@avro_dec_old +PREHOOK: Output: default@avro_dec_old +POSTHOOK: query: DROP TABLE avro_dec_old +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@avro_dec_old +POSTHOOK: Output: default@avro_dec_old http://git-wip-us.apache.org/repos/asf/hive/blob/d4c1fdcf/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java ---------------------------------------------------------------------- diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java index 4bba3d4..6165138 100644 --- a/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java +++ b/serde/src/java/org/apache/hadoop/hive/serde2/avro/AvroDeserializer.java @@ -244,7 +244,7 @@ class AvroDeserializer { int scale = 0; try { - scale = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_SCALE).getIntValue(); + scale = fileSchema.getJsonProp(AvroSerDe.AVRO_PROP_SCALE).asInt(); } catch(Exception ex) { throw new AvroSerdeException("Failed to obtain scale value from file schema: " + fileSchema, ex); }
