This is an automated email from the ASF dual-hosted git repository.
kuczoram pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8111ede HIVE-21987: Hive is unable to read Parquet int32 annotated
with decimal (Marta Kuczora reviewed by Peter Vary)
8111ede is described below
commit 8111edec6de1d8d93c28471163ca40eae377129d
Author: Marta Kuczora <[email protected]>
AuthorDate: Tue Oct 1 13:10:48 2019 +0200
HIVE-21987: Hive is unable to read Parquet int32 annotated with decimal
(Marta Kuczora reviewed by Peter Vary)
---
data/files/parquet_int_decimal_1.parquet | Bin 0 -> 1151 bytes
data/files/parquet_int_decimal_2.parquet | Bin 0 -> 1128 bytes
.../hive/ql/io/parquet/convert/ETypeConverter.java | 121 +++--
.../vector/ParquetDataColumnReaderFactory.java | 333 ++++++++++++++
.../queries/clientpositive/parquet_int_decimal.q | 62 +++
.../clientpositive/parquet_int_decimal.q.out | 489 +++++++++++++++++++++
.../clientpositive/type_change_test_fraction.q.out | 16 +-
7 files changed, 975 insertions(+), 46 deletions(-)
diff --git a/data/files/parquet_int_decimal_1.parquet
b/data/files/parquet_int_decimal_1.parquet
new file mode 100644
index 0000000..56463a9
Binary files /dev/null and b/data/files/parquet_int_decimal_1.parquet differ
diff --git a/data/files/parquet_int_decimal_2.parquet
b/data/files/parquet_int_decimal_2.parquet
new file mode 100644
index 0000000..ee4f36b
Binary files /dev/null and b/data/files/parquet_int_decimal_2.parquet differ
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java
index 350ae2d..89dfe2d 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/ETypeConverter.java
@@ -17,6 +17,7 @@ import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.Map;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport;
@@ -494,11 +495,25 @@ public enum ETypeConverter {
public void addBinary(Binary value) {
HiveDecimalWritable decimalWritable =
new HiveDecimalWritable(value.getBytes(),
type.getDecimalMetadata().getScale());
- double doubleValue = decimalWritable.doubleValue();
- double absDoubleValue = (doubleValue < 0) ? (doubleValue * -1) :
doubleValue;
+ setValue(decimalWritable.doubleValue(),
decimalWritable.floatValue());
+ }
- if ((absDoubleValue >= minValue) && (absDoubleValue <=
maxValue)) {
- parent.set(index, new
FloatWritable(decimalWritable.floatValue()));
+ @Override
+ public void addInt(final int value) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(value,
type.getDecimalMetadata().getScale());
+ setValue(hiveDecimal.doubleValue(), hiveDecimal.floatValue());
+ }
+
+ @Override
+ public void addLong(final long value) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(value,
type.getDecimalMetadata().getScale());
+ setValue(hiveDecimal.doubleValue(), hiveDecimal.floatValue());
+ }
+
+ private void setValue(double doubleValue, float floatValue) {
+ double absDoubleValue = (doubleValue < 0) ? (doubleValue * -1) :
doubleValue;
+ if (((absDoubleValue >= minValue) && (absDoubleValue <=
maxValue)) || absDoubleValue == 0d) {
+ parent.set(index, new FloatWritable(floatValue));
} else {
parent.set(index, null);
}
@@ -512,6 +527,18 @@ public enum ETypeConverter {
new HiveDecimalWritable(value.getBytes(),
type.getDecimalMetadata().getScale());
parent.set(index, new
DoubleWritable(decimalWritable.doubleValue()));
}
+
+ @Override
+ public void addInt(final int value) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(value,
type.getDecimalMetadata().getScale());
+ parent.set(index, new DoubleWritable(hiveDecimal.doubleValue()));
+ }
+
+ @Override
+ public void addLong(final long value) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(value,
type.getDecimalMetadata().getScale());
+ parent.set(index, new DoubleWritable(hiveDecimal.doubleValue()));
+ }
};
case serdeConstants.BIGINT_TYPE_NAME:
return new PrimitiveConverter() {
@@ -519,55 +546,55 @@ public enum ETypeConverter {
public void addBinary(Binary value) {
HiveDecimalWritable decimalWritable =
new HiveDecimalWritable(value.getBytes(),
type.getDecimalMetadata().getScale());
- double doubleValue = decimalWritable.doubleValue();
- if ((doubleValue >= minValue) && (doubleValue <= maxValue) &&
- (doubleValue % 1 == 0)) {
- parent.set(index, new
LongWritable(decimalWritable.longValue()));
- } else {
- parent.set(index, null);
- }
+ setValue(decimalWritable.doubleValue(),
decimalWritable.longValue());
}
- };
- case serdeConstants.INT_TYPE_NAME:
- return new PrimitiveConverter() {
+
@Override
- public void addBinary(Binary value) {
- HiveDecimalWritable decimalWritable =
- new HiveDecimalWritable(value.getBytes(),
type.getDecimalMetadata().getScale());
- double doubleValue = decimalWritable.doubleValue();
- if ((doubleValue >= minValue) && (doubleValue <= maxValue) &&
- (doubleValue % 1 == 0)) {
- parent.set(index, new IntWritable(decimalWritable.intValue()));
- } else {
- parent.set(index, null);
- }
+ public void addInt(final int value) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(value,
type.getDecimalMetadata().getScale());
+ setValue(hiveDecimal.doubleValue(), hiveDecimal.longValue());
}
- };
- case serdeConstants.SMALLINT_TYPE_NAME:
- return new PrimitiveConverter() {
+
@Override
- public void addBinary(Binary value) {
- HiveDecimalWritable decimalWritable =
- new HiveDecimalWritable(value.getBytes(),
type.getDecimalMetadata().getScale());
- double doubleValue = decimalWritable.doubleValue();
- if ((doubleValue >= minValue) && (doubleValue <= maxValue) &&
- (doubleValue % 1 == 0)) {
- parent.set(index, new IntWritable(decimalWritable.intValue()));
+ public void addLong(final long value) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(value,
type.getDecimalMetadata().getScale());
+ setValue(hiveDecimal.doubleValue(), hiveDecimal.longValue());
+ }
+
+ private void setValue(double doubleValue, long longValue) {
+ if ((doubleValue >= minValue) && (doubleValue <= maxValue) &&
(doubleValue % 1 == 0)) {
+ parent.set(index, new LongWritable(longValue));
} else {
parent.set(index, null);
}
}
};
+ case serdeConstants.INT_TYPE_NAME:
+ case serdeConstants.SMALLINT_TYPE_NAME:
case serdeConstants.TINYINT_TYPE_NAME:
return new PrimitiveConverter() {
@Override
public void addBinary(Binary value) {
HiveDecimalWritable decimalWritable =
new HiveDecimalWritable(value.getBytes(),
type.getDecimalMetadata().getScale());
- double doubleValue = decimalWritable.doubleValue();
- if ((doubleValue >= minValue) && (doubleValue <= maxValue) &&
- (doubleValue % 1 == 0)) {
- parent.set(index, new IntWritable(decimalWritable.intValue()));
+ setValue(decimalWritable.doubleValue(),
decimalWritable.intValue());
+ }
+
+ @Override
+ public void addInt(final int value) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(value,
type.getDecimalMetadata().getScale());
+ setValue(hiveDecimal.doubleValue(), hiveDecimal.intValue());
+ }
+
+ @Override
+ public void addLong(final long value) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(value,
type.getDecimalMetadata().getScale());
+ setValue(hiveDecimal.doubleValue(), hiveDecimal.intValue());
+ }
+
+ private void setValue(double doubleValue, int intValue) {
+ if ((doubleValue >= minValue) && (doubleValue <= maxValue) &&
(doubleValue % 1 == 0)) {
+ parent.set(index, new IntWritable(intValue));
} else {
parent.set(index, null);
}
@@ -581,6 +608,24 @@ public enum ETypeConverter {
new HiveDecimalWritable(binary.getBytes(),
type.getDecimalMetadata().getScale()),
(DecimalTypeInfo) hiveTypeInfo);
}
+
+ @Override
+ public void addInt(final int value) {
+ addDecimal(value);
+ }
+
+ @Override
+ public void addLong(final long value) {
+ addDecimal(value);
+ }
+
+ private void addDecimal(long value) {
+ DecimalTypeInfo decimalInfo = (DecimalTypeInfo) hiveTypeInfo;
+ HiveDecimal hiveDecimal = HiveDecimal.create(value,
decimalInfo.scale());
+ HiveDecimalWritable result =
HiveDecimalUtils.enforcePrecisionScale(new HiveDecimalWritable(hiveDecimal),
+ (DecimalTypeInfo) hiveTypeInfo);
+ parent.set(index, result);
+ }
};
}
}
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java
index 320ce52..979ef47 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/vector/ParquetDataColumnReaderFactory.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hive.ql.io.parquet.vector;
import org.apache.hadoop.hive.common.type.HiveBaseChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.ql.exec.vector.expressions.StringExpr;
import org.apache.hadoop.hive.ql.io.parquet.timestamp.NanoTime;
@@ -1421,6 +1422,330 @@ public final class ParquetDataColumnReaderFactory {
}
/**
+ * The reader who reads from the underlying decimal value which is stored in
an INT32 physical type.
+ *
+ * The data is read as INT32 from the reader treated as a decimal, then
validated, converted
+ * and returned as per the type defined in HMS.
+ */
+ public static class TypesFromInt32DecimalPageReader extends
DefaultParquetDataColumnReader {
+ private short scale;
+
+ public TypesFromInt32DecimalPageReader(ValuesReader realReader, int
length, short scale, int hivePrecision,
+ int hiveScale) {
+ super(realReader, length, hivePrecision, hiveScale);
+ this.scale = scale;
+ }
+
+ public TypesFromInt32DecimalPageReader(Dictionary dict, int length, short
scale, int hivePrecision, int hiveScale) {
+ super(dict, length, hivePrecision, hiveScale);
+ this.scale = scale;
+ }
+
+ @Override
+ public byte[] readString() {
+ return convertToBytes(valuesReader.readInteger());
+ }
+
+ @Override
+ public byte[] readString(int id) {
+ return convertToBytes(dict.decodeToInt(id));
+ }
+
+ @Override
+ public byte[] readVarchar() {
+ String value =
enforceMaxLength(convertToString(valuesReader.readInteger()));
+ return convertToBytes(value);
+ }
+
+ @Override
+ public byte[] readVarchar(int id) {
+ String value = enforceMaxLength(convertToString(dict.decodeToInt(id)));
+ return convertToBytes(value);
+ }
+
+ @Override
+ public byte[] readChar() {
+ String value =
enforceMaxLength(convertToString(valuesReader.readInteger()));
+ return convertToBytes(value);
+ }
+
+ @Override
+ public byte[] readChar(int id) {
+ String value = enforceMaxLength(convertToString(dict.decodeToInt(id)));
+ return convertToBytes(value);
+ }
+
+ @Override
+ public float readFloat() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readInteger(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (float) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.FLOAT_TYPE_NAME));
+ }
+
+ @Override
+ public float readFloat(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToInt(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (float) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.FLOAT_TYPE_NAME));
+ }
+
+ @Override
+ public double readDouble() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readInteger(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.DOUBLE_TYPE_NAME));
+ }
+
+ @Override
+ public double readDouble(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToInt(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.DOUBLE_TYPE_NAME));
+ }
+
+ @Override
+ public long readLong() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readInteger(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.BIGINT_TYPE_NAME));
+ }
+
+ @Override
+ public long readLong(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToInt(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.BIGINT_TYPE_NAME));
+ }
+
+ @Override
+ public long readInteger() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readInteger(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.INT_TYPE_NAME));
+ }
+
+ @Override
+ public long readInteger(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToInt(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.INT_TYPE_NAME));
+ }
+
+ @Override
+ public long readSmallInt() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readInteger(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.SMALLINT_TYPE_NAME));
+ }
+
+ @Override
+ public long readSmallInt(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToInt(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.SMALLINT_TYPE_NAME));
+ }
+
+ @Override
+ public long readTinyInt() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readInteger(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.TINYINT_TYPE_NAME));
+ }
+
+ @Override
+ public long readTinyInt(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToInt(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.TINYINT_TYPE_NAME));
+ }
+
+ private String convertToString(int value) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(value, scale);
+ return hiveDecimal.toString();
+ }
+
+ private byte[] convertToBytes(int value) {
+ return convertToBytes(convertToString(value));
+ }
+
+ @Override
+ public byte[] readDecimal() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readInteger(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return super.validatedScaledDecimal(scale);
+ }
+
+ @Override
+ public byte[] readDecimal(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToInt(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return super.validatedScaledDecimal(scale);
+ }
+ }
+
+ /**
+ * The reader who reads from the underlying decimal value which is stored in
an INT64 physical type.
+ *
+ * The data is read as INT64 from the reader treated as a decimal, then
validated, converted
+ * and returned as per the type defined in HMS.
+ */
+ public static class TypesFromInt64DecimalPageReader extends
DefaultParquetDataColumnReader {
+ private short scale;
+
+ public TypesFromInt64DecimalPageReader(ValuesReader realReader, int
length, short scale, int hivePrecision,
+ int hiveScale) {
+ super(realReader, length, hivePrecision, hiveScale);
+ this.scale = scale;
+ }
+
+ public TypesFromInt64DecimalPageReader(Dictionary dict, int length, short
scale, int hivePrecision, int hiveScale) {
+ super(dict, length, hivePrecision, hiveScale);
+ this.scale = scale;
+ }
+
+ @Override
+ public byte[] readString() {
+ return convertToBytes(valuesReader.readLong());
+ }
+
+ @Override
+ public byte[] readString(int id) {
+ return convertToBytes(dict.decodeToLong(id));
+ }
+
+ @Override
+ public byte[] readVarchar() {
+ String value =
enforceMaxLength(convertToString(valuesReader.readLong()));
+ return convertToBytes(value);
+ }
+
+ @Override
+ public byte[] readVarchar(int id) {
+ String value = enforceMaxLength(convertToString(dict.decodeToLong(id)));
+ return convertToBytes(value);
+ }
+
+ @Override
+ public byte[] readChar() {
+ String value =
enforceMaxLength(convertToString(valuesReader.readLong()));
+ return convertToBytes(value);
+ }
+
+ @Override
+ public byte[] readChar(int id) {
+ String value = enforceMaxLength(convertToString(dict.decodeToLong(id)));
+ return convertToBytes(value);
+ }
+
+ @Override
+ public float readFloat() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readLong(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (float) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.FLOAT_TYPE_NAME));
+ }
+
+ @Override
+ public float readFloat(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToLong(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (float) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.FLOAT_TYPE_NAME));
+ }
+
+ @Override
+ public double readDouble() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readLong(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.DOUBLE_TYPE_NAME));
+ }
+
+ @Override
+ public double readDouble(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToLong(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.DOUBLE_TYPE_NAME));
+ }
+
+ @Override
+ public long readLong() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readLong(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.BIGINT_TYPE_NAME));
+ }
+
+ @Override
+ public long readLong(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToLong(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.BIGINT_TYPE_NAME));
+ }
+
+ @Override
+ public long readInteger() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readLong(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.INT_TYPE_NAME));
+ }
+
+ @Override
+ public long readInteger(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToLong(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.INT_TYPE_NAME));
+ }
+
+ @Override
+ public long readSmallInt() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readLong(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.SMALLINT_TYPE_NAME));
+ }
+
+ @Override
+ public long readSmallInt(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToLong(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.SMALLINT_TYPE_NAME));
+ }
+
+ @Override
+ public long readTinyInt() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readLong(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.TINYINT_TYPE_NAME));
+ }
+
+ @Override
+ public long readTinyInt(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToLong(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return (long) (super.validatedDouble(hiveDecimalWritable.doubleValue(),
serdeConstants.TINYINT_TYPE_NAME));
+ }
+
+ private String convertToString(long value) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(value, scale);
+ return hiveDecimal.toString();
+ }
+
+ private byte[] convertToBytes(long value) {
+ return convertToBytes(convertToString(value));
+ }
+
+ @Override
+ public byte[] readDecimal() {
+ HiveDecimal hiveDecimal = HiveDecimal.create(valuesReader.readLong(),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return super.validatedScaledDecimal(scale);
+ }
+
+ @Override
+ public byte[] readDecimal(int id) {
+ HiveDecimal hiveDecimal = HiveDecimal.create(dict.decodeToLong(id),
scale);
+ hiveDecimalWritable.set(hiveDecimal);
+ return super.validatedScaledDecimal(scale);
+ }
+ }
+
+ /**
* The reader who reads from the underlying UTF8 string.
*/
public static class TypesFromStringPageReader extends
DefaultParquetDataColumnReader {
@@ -1506,6 +1831,10 @@ public final class ParquetDataColumnReaderFactory {
return isDictionary ? new TypesFromUInt32PageReader(dictionary,
length, hivePrecision,
hiveScale) : new TypesFromUInt32PageReader(valuesReader, length,
hivePrecision,
hiveScale);
+ } else if (OriginalType.DECIMAL == parquetType.getOriginalType()) {
+ final short scale = (short)
parquetType.asPrimitiveType().getDecimalMetadata().getScale();
+ return isDictionary ? new TypesFromInt32DecimalPageReader(dictionary,
length, scale, hivePrecision, hiveScale)
+ : new TypesFromInt32DecimalPageReader(valuesReader, length, scale,
hivePrecision, hiveScale);
} else {
return isDictionary ? new TypesFromInt32PageReader(dictionary, length,
hivePrecision,
hiveScale) : new TypesFromInt32PageReader(valuesReader, length,
hivePrecision,
@@ -1519,6 +1848,10 @@ public final class ParquetDataColumnReaderFactory {
return isDictionary ? new TypesFromUInt64PageReader(dictionary,
length, hivePrecision,
hiveScale) : new TypesFromUInt64PageReader(valuesReader, length,
hivePrecision,
hiveScale);
+ } else if (OriginalType.DECIMAL == parquetType.getOriginalType()) {
+ final short scale = (short)
parquetType.asPrimitiveType().getDecimalMetadata().getScale();
+ return isDictionary ? new TypesFromInt64DecimalPageReader(dictionary,
length, scale, hivePrecision, hiveScale)
+ : new TypesFromInt64DecimalPageReader(valuesReader, length, scale,
hivePrecision, hiveScale);
} else {
return isDictionary ? new TypesFromInt64PageReader(dictionary, length,
hivePrecision,
hiveScale) : new TypesFromInt64PageReader(valuesReader, length,
hivePrecision,
diff --git a/ql/src/test/queries/clientpositive/parquet_int_decimal.q
b/ql/src/test/queries/clientpositive/parquet_int_decimal.q
new file mode 100644
index 0000000..4b292e2
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/parquet_int_decimal.q
@@ -0,0 +1,62 @@
+set hive.vectorized.execution.enabled=false;
+
+DROP TABLE parquet_decimal_spark;
+DROP TABLE parquet_decimal_bigint_spark;
+DROP TABLE parquet_decimal_int_spark;
+DROP TABLE parquet_decimal_smallint_spark;
+DROP TABLE parquet_decimal_tinyint_spark;
+DROP TABLE parquet_decimal_double_spark;
+DROP TABLE parquet_decimal_float_spark;
+
+CREATE TABLE parquet_decimal_spark (d1 decimal(4,2), d2 decimal(4,0), d3
decimal(12,5), d4 decimal(12,0)) stored as parquet;
+CREATE TABLE parquet_decimal_bigint_spark (d1 bigint, d2 bigint, d3 bigint, d4
bigint) stored as parquet;
+CREATE TABLE parquet_decimal_int_spark (d1 int, d2 int, d3 int, d4 int) stored
as parquet;
+CREATE TABLE parquet_decimal_smallint_spark (d1 smallint, d2 smallint, d3
smallint, d4 smallint) stored as parquet;
+CREATE TABLE parquet_decimal_tinyint_spark (d1 tinyint, d2 tinyint, d3
tinyint, d4 tinyint) stored as parquet;
+CREATE TABLE parquet_decimal_double_spark (d1 double, d2 double, d3 double, d4
double) stored as parquet;
+CREATE TABLE parquet_decimal_float_spark (d1 float, d2 float, d3 float, d4
float) stored as parquet;
+CREATE TABLE parquet_decimal_string_spark (d1 string, d2 string, d3 string, d4
string) stored as parquet;
+CREATE TABLE parquet_decimal_varchar_spark (d1 varchar(100), d2 varchar(100),
d3 varchar(100), d4 varchar(100)) stored as parquet;
+CREATE TABLE parquet_decimal_char_spark (d1 char(4), d2 char(4), d3 char(4),
d4 char(4)) stored as parquet;
+
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_1.parquet' INTO
TABLE parquet_decimal_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_2.parquet' INTO
TABLE parquet_decimal_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_1.parquet' INTO
TABLE parquet_decimal_bigint_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_2.parquet' INTO
TABLE parquet_decimal_bigint_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_1.parquet' INTO
TABLE parquet_decimal_int_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_2.parquet' INTO
TABLE parquet_decimal_int_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_1.parquet' INTO
TABLE parquet_decimal_smallint_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_2.parquet' INTO
TABLE parquet_decimal_smallint_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_1.parquet' INTO
TABLE parquet_decimal_tinyint_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_2.parquet' INTO
TABLE parquet_decimal_tinyint_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_1.parquet' INTO
TABLE parquet_decimal_double_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_2.parquet' INTO
TABLE parquet_decimal_double_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_1.parquet' INTO
TABLE parquet_decimal_float_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_2.parquet' INTO
TABLE parquet_decimal_float_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_1.parquet' INTO
TABLE parquet_decimal_string_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_2.parquet' INTO
TABLE parquet_decimal_string_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_1.parquet' INTO
TABLE parquet_decimal_varchar_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_2.parquet' INTO
TABLE parquet_decimal_varchar_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_1.parquet' INTO
TABLE parquet_decimal_char_spark;
+LOAD DATA LOCAL INPATH '../../data/files/parquet_int_decimal_2.parquet' INTO
TABLE parquet_decimal_char_spark;
+
+SELECT * FROM parquet_decimal_spark;
+SELECT * FROM parquet_decimal_bigint_spark;
+SELECT * FROM parquet_decimal_int_spark;
+SELECT * FROM parquet_decimal_smallint_spark;
+SELECT * FROM parquet_decimal_tinyint_spark;
+SELECT * FROM parquet_decimal_double_spark;
+SELECT * FROM parquet_decimal_float_spark;
+
+set hive.vectorized.execution.enabled=true;
+
+SELECT * FROM parquet_decimal_spark order by d2;
+SELECT * FROM parquet_decimal_bigint_spark order by d2;
+SELECT * FROM parquet_decimal_int_spark order by d2;
+SELECT * FROM parquet_decimal_smallint_spark order by d2;
+SELECT * FROM parquet_decimal_tinyint_spark order by d2;
+SELECT * FROM parquet_decimal_double_spark order by d2;
+SELECT * FROM parquet_decimal_float_spark order by d2;
+SELECT * FROM parquet_decimal_string_spark order by d2;
+SELECT * FROM parquet_decimal_varchar_spark order by d2;
+SELECT * FROM parquet_decimal_char_spark order by d2;
diff --git a/ql/src/test/results/clientpositive/parquet_int_decimal.q.out
b/ql/src/test/results/clientpositive/parquet_int_decimal.q.out
new file mode 100644
index 0000000..6586edf
--- /dev/null
+++ b/ql/src/test/results/clientpositive/parquet_int_decimal.q.out
@@ -0,0 +1,489 @@
+PREHOOK: query: DROP TABLE parquet_decimal_spark
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_decimal_spark
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE parquet_decimal_bigint_spark
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_decimal_bigint_spark
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE parquet_decimal_int_spark
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_decimal_int_spark
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE parquet_decimal_smallint_spark
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_decimal_smallint_spark
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE parquet_decimal_tinyint_spark
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_decimal_tinyint_spark
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE parquet_decimal_double_spark
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_decimal_double_spark
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE parquet_decimal_float_spark
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE parquet_decimal_float_spark
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE parquet_decimal_spark (d1 decimal(4,2), d2
decimal(4,0), d3 decimal(12,5), d4 decimal(12,0)) stored as parquet
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_decimal_spark
+POSTHOOK: query: CREATE TABLE parquet_decimal_spark (d1 decimal(4,2), d2
decimal(4,0), d3 decimal(12,5), d4 decimal(12,0)) stored as parquet
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_decimal_spark
+PREHOOK: query: CREATE TABLE parquet_decimal_bigint_spark (d1 bigint, d2
bigint, d3 bigint, d4 bigint) stored as parquet
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_decimal_bigint_spark
+POSTHOOK: query: CREATE TABLE parquet_decimal_bigint_spark (d1 bigint, d2
bigint, d3 bigint, d4 bigint) stored as parquet
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_decimal_bigint_spark
+PREHOOK: query: CREATE TABLE parquet_decimal_int_spark (d1 int, d2 int, d3
int, d4 int) stored as parquet
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_decimal_int_spark
+POSTHOOK: query: CREATE TABLE parquet_decimal_int_spark (d1 int, d2 int, d3
int, d4 int) stored as parquet
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_decimal_int_spark
+PREHOOK: query: CREATE TABLE parquet_decimal_smallint_spark (d1 smallint, d2
smallint, d3 smallint, d4 smallint) stored as parquet
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_decimal_smallint_spark
+POSTHOOK: query: CREATE TABLE parquet_decimal_smallint_spark (d1 smallint, d2
smallint, d3 smallint, d4 smallint) stored as parquet
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_decimal_smallint_spark
+PREHOOK: query: CREATE TABLE parquet_decimal_tinyint_spark (d1 tinyint, d2
tinyint, d3 tinyint, d4 tinyint) stored as parquet
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_decimal_tinyint_spark
+POSTHOOK: query: CREATE TABLE parquet_decimal_tinyint_spark (d1 tinyint, d2
tinyint, d3 tinyint, d4 tinyint) stored as parquet
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_decimal_tinyint_spark
+PREHOOK: query: CREATE TABLE parquet_decimal_double_spark (d1 double, d2
double, d3 double, d4 double) stored as parquet
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_decimal_double_spark
+POSTHOOK: query: CREATE TABLE parquet_decimal_double_spark (d1 double, d2
double, d3 double, d4 double) stored as parquet
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_decimal_double_spark
+PREHOOK: query: CREATE TABLE parquet_decimal_float_spark (d1 float, d2 float,
d3 float, d4 float) stored as parquet
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_decimal_float_spark
+POSTHOOK: query: CREATE TABLE parquet_decimal_float_spark (d1 float, d2 float,
d3 float, d4 float) stored as parquet
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_decimal_float_spark
+PREHOOK: query: CREATE TABLE parquet_decimal_string_spark (d1 string, d2
string, d3 string, d4 string) stored as parquet
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_decimal_string_spark
+POSTHOOK: query: CREATE TABLE parquet_decimal_string_spark (d1 string, d2
string, d3 string, d4 string) stored as parquet
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_decimal_string_spark
+PREHOOK: query: CREATE TABLE parquet_decimal_varchar_spark (d1 varchar(100),
d2 varchar(100), d3 varchar(100), d4 varchar(100)) stored as parquet
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_decimal_varchar_spark
+POSTHOOK: query: CREATE TABLE parquet_decimal_varchar_spark (d1 varchar(100),
d2 varchar(100), d3 varchar(100), d4 varchar(100)) stored as parquet
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_decimal_varchar_spark
+PREHOOK: query: CREATE TABLE parquet_decimal_char_spark (d1 char(4), d2
char(4), d3 char(4), d4 char(4)) stored as parquet
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@parquet_decimal_char_spark
+POSTHOOK: query: CREATE TABLE parquet_decimal_char_spark (d1 char(4), d2
char(4), d3 char(4), d4 char(4)) stored as parquet
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@parquet_decimal_char_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_bigint_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_bigint_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_bigint_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_bigint_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_bigint_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_bigint_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_bigint_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_bigint_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_int_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_int_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_int_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_int_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_int_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_int_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_int_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_int_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_smallint_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_smallint_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_smallint_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_smallint_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_smallint_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_smallint_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_smallint_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_smallint_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_tinyint_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_tinyint_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_tinyint_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_tinyint_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_tinyint_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_tinyint_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_tinyint_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_tinyint_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_double_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_double_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_double_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_double_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_double_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_double_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_double_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_double_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_float_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_float_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_float_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_float_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_float_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_float_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_float_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_float_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_string_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_string_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_string_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_string_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_string_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_string_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_string_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_string_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_varchar_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_varchar_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_varchar_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_varchar_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_varchar_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_varchar_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_varchar_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_varchar_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_char_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_char_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_1.parquet' INTO TABLE
parquet_decimal_char_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_char_spark
+PREHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_char_spark
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@parquet_decimal_char_spark
+POSTHOOK: query: LOAD DATA LOCAL INPATH
'../../data/files/parquet_int_decimal_2.parquet' INTO TABLE
parquet_decimal_char_spark
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@parquet_decimal_char_spark
+PREHOOK: query: SELECT * FROM parquet_decimal_spark
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_spark
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_spark
+#### A masked pattern was here ####
+44.21 54 3344556.21211 3344556
+52.00 62 8877665.00000 776644993322
+0.00 0 0.00000 0
+-42.25 -52 -2233114.88776 -2233115
+-15.00 -25 -8877665.00000 -776644993322
+PREHOOK: query: SELECT * FROM parquet_decimal_bigint_spark
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_bigint_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_bigint_spark
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_bigint_spark
+#### A masked pattern was here ####
+NULL 54 NULL 3344556
+52 62 8877665 776644993322
+0 0 0 0
+NULL -52 NULL -2233115
+-15 -25 -8877665 -776644993322
+PREHOOK: query: SELECT * FROM parquet_decimal_int_spark
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_int_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_int_spark
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_int_spark
+#### A masked pattern was here ####
+NULL 54 NULL 3344556
+52 62 8877665 NULL
+0 0 0 0
+NULL -52 NULL -2233115
+-15 -25 -8877665 NULL
+PREHOOK: query: SELECT * FROM parquet_decimal_smallint_spark
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_smallint_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_smallint_spark
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_smallint_spark
+#### A masked pattern was here ####
+NULL 54 NULL NULL
+52 62 NULL NULL
+0 0 0 0
+NULL -52 NULL NULL
+-15 -25 NULL NULL
+PREHOOK: query: SELECT * FROM parquet_decimal_tinyint_spark
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_tinyint_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_tinyint_spark
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_tinyint_spark
+#### A masked pattern was here ####
+NULL 54 NULL NULL
+52 62 NULL NULL
+0 0 0 0
+NULL -52 NULL NULL
+-15 -25 NULL NULL
+PREHOOK: query: SELECT * FROM parquet_decimal_double_spark
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_double_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_double_spark
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_double_spark
+#### A masked pattern was here ####
+44.21 54.0 3344556.21211 3344556.0
+52.0 62.0 8877665.0 7.76644993322E11
+0.0 0.0 0.0 0.0
+-42.25 -52.0 -2233114.88776 -2233115.0
+-15.0 -25.0 -8877665.0 -7.76644993322E11
+PREHOOK: query: SELECT * FROM parquet_decimal_float_spark
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_float_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_float_spark
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_float_spark
+#### A masked pattern was here ####
+44.21 54.0 3344556.2 3344556.0
+52.0 62.0 8877665.0 7.7664498E11
+0.0 0.0 0.0 0.0
+-42.25 -52.0 -2233115.0 -2233115.0
+-15.0 -25.0 -8877665.0 -7.7664498E11
+PREHOOK: query: SELECT * FROM parquet_decimal_spark order by d2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_spark order by d2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_spark
+#### A masked pattern was here ####
+-42.25 -52 -2233114.88776 -2233115
+-15.00 -25 -8877665.00000 -776644993322
+0.00 0 0.00000 0
+44.21 54 3344556.21211 3344556
+52.00 62 8877665.00000 776644993322
+PREHOOK: query: SELECT * FROM parquet_decimal_bigint_spark order by d2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_bigint_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_bigint_spark order by d2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_bigint_spark
+#### A masked pattern was here ####
+NULL -52 NULL -2233115
+-15 -25 -8877665 -776644993322
+0 0 0 0
+NULL 54 NULL 3344556
+52 62 8877665 776644993322
+PREHOOK: query: SELECT * FROM parquet_decimal_int_spark order by d2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_int_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_int_spark order by d2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_int_spark
+#### A masked pattern was here ####
+NULL -52 NULL -2233115
+-15 -25 -8877665 NULL
+0 0 0 0
+NULL 54 NULL 3344556
+52 62 8877665 NULL
+PREHOOK: query: SELECT * FROM parquet_decimal_smallint_spark order by d2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_smallint_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_smallint_spark order by d2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_smallint_spark
+#### A masked pattern was here ####
+NULL -52 NULL NULL
+-15 -25 NULL NULL
+0 0 0 0
+NULL 54 NULL NULL
+52 62 NULL NULL
+PREHOOK: query: SELECT * FROM parquet_decimal_tinyint_spark order by d2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_tinyint_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_tinyint_spark order by d2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_tinyint_spark
+#### A masked pattern was here ####
+NULL -52 NULL NULL
+-15 -25 NULL NULL
+0 0 0 0
+NULL 54 NULL NULL
+52 62 NULL NULL
+PREHOOK: query: SELECT * FROM parquet_decimal_double_spark order by d2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_double_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_double_spark order by d2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_double_spark
+#### A masked pattern was here ####
+-42.25 -52.0 -2233114.88776 -2233115.0
+-15.0 -25.0 -8877665.0 -7.76644993322E11
+0.0 0.0 0.0 0.0
+44.21 54.0 3344556.21211 3344556.0
+52.0 62.0 8877665.0 7.76644993322E11
+PREHOOK: query: SELECT * FROM parquet_decimal_float_spark order by d2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_float_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_float_spark order by d2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_float_spark
+#### A masked pattern was here ####
+-42.25 -52.0 -2233115.0 -2233115.0
+-15.0 -25.0 -8877665.0 -7.7664498E11
+44.21 54.0 3344556.2 3344556.0
+52.0 62.0 8877665.0 7.7664498E11
+NULL NULL NULL NULL
+PREHOOK: query: SELECT * FROM parquet_decimal_string_spark order by d2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_string_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_string_spark order by d2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_string_spark
+#### A masked pattern was here ####
+-15 -25 -8877665 -776644993322
+-42.25 -52 -2233114.88776 -2233115
+0 0 0 0
+44.21 54 3344556.21211 3344556
+52 62 8877665 776644993322
+PREHOOK: query: SELECT * FROM parquet_decimal_varchar_spark order by d2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_varchar_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_varchar_spark order by d2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_varchar_spark
+#### A masked pattern was here ####
+-15 -25 -8877665 -776644993322
+-42.25 -52 -2233114.88776 -2233115
+0 0 0 0
+44.21 54 3344556.21211 3344556
+52 62 8877665 776644993322
+PREHOOK: query: SELECT * FROM parquet_decimal_char_spark order by d2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@parquet_decimal_char_spark
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM parquet_decimal_char_spark order by d2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@parquet_decimal_char_spark
+#### A masked pattern was here ####
+-15 -25 -887 -776
+-42. -52 -223 -223
+0 0 0 0
+44.2 54 3344 3344
+52 62 8877 7766
diff --git a/ql/src/test/results/clientpositive/type_change_test_fraction.q.out
b/ql/src/test/results/clientpositive/type_change_test_fraction.q.out
index 07cf8fa..4a2ee3f 100644
--- a/ql/src/test/results/clientpositive/type_change_test_fraction.q.out
+++ b/ql/src/test/results/clientpositive/type_change_test_fraction.q.out
@@ -7180,8 +7180,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@testaltcolpde_n2
#### A masked pattern was here ####
1 1.2345679E19 1.2345679 1.2345679E7 1.23
-2 1.0E-18 NULL 1.0E-8 0.01
-3 -1.0E-18 NULL -1.0E-8 -0.01
+2 1.0E-18 0.0 1.0E-8 0.01
+3 -1.0E-18 0.0 -1.0E-8 -0.01
4 1.0E20 NULL 1.0E8 9.99
5 -1.0E20 NULL -1.0E8 -9.99
6 1.0 1.0 1.0 1.0
@@ -7209,8 +7209,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@testaltcolpde_n2
#### A masked pattern was here ####
1 1.2345679E19 1.2345679 1.2345679E7 1.23
-2 1.0E-18 NULL 1.0E-8 0.01
-3 -1.0E-18 NULL -1.0E-8 -0.01
+2 1.0E-18 0.0 1.0E-8 0.01
+3 -1.0E-18 0.0 -1.0E-8 -0.01
4 1.0E20 NULL 1.0E8 9.99
5 -1.0E20 NULL -1.0E8 -9.99
6 1.0 1.0 1.0 1.0
@@ -8910,8 +8910,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@testaltcolpdd_n2
#### A masked pattern was here ####
1 1.2345679E19 1.2345679 1.2345679E7 1.23
-2 1.0E-18 NULL 1.0E-8 0.01
-3 -1.0E-18 NULL -1.0E-8 -0.01
+2 1.0E-18 0.0 1.0E-8 0.01
+3 -1.0E-18 0.0 -1.0E-8 -0.01
4 1.0E20 NULL 1.0E8 9.99
5 -1.0E20 NULL -1.0E8 -9.99
6 1.0 1.0 1.0 1.0
@@ -8939,8 +8939,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@testaltcolpdd_n2
#### A masked pattern was here ####
1 1.2345679E19 1.2345679 1.2345679E7 1.23
-2 1.0E-18 NULL 1.0E-8 0.01
-3 -1.0E-18 NULL -1.0E-8 -0.01
+2 1.0E-18 0.0 1.0E-8 0.01
+3 -1.0E-18 0.0 -1.0E-8 -0.01
4 1.0E20 NULL 1.0E8 9.99
5 -1.0E20 NULL -1.0E8 -9.99
6 1.0 1.0 1.0 1.0
