DRILL-5097: Using store.parquet.reader.int96_as_timestamp gives IOOB whereas convert_from works
close apache/drill#697 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/08ca5e09 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/08ca5e09 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/08ca5e09 Branch: refs/heads/master Commit: 08ca5e0923436ebeb3e190c60892167d9beab0c2 Parents: bb29f19 Author: Vitalii Diravka <vitalii.dira...@gmail.com> Authored: Wed Dec 14 16:24:08 2016 +0000 Committer: Jinfeng Ni <j...@apache.org> Committed: Mon Jan 23 17:08:20 2017 -0800 ---------------------------------------------------------------------- .../NullableFixedByteAlignedReaders.java | 10 +++++++++- .../physical/impl/writer/TestParquetWriter.java | 19 +++++++++++++++++++ .../test/resources/parquet/data.snappy.parquet | Bin 0 -> 59200 bytes 3 files changed, 28 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/08ca5e09/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java index e20504f..b233a65 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/columnreaders/NullableFixedByteAlignedReaders.java @@ -110,9 +110,14 @@ public class NullableFixedByteAlignedReaders { /** * Class for reading parquet fixed binary type INT96, which is used for storing hive, - * impala timestamp values with nanoseconds precision. So it reads such values as a drill timestamp. + * impala timestamp values with nanoseconds precision (12 bytes). So it reads such values as a drill timestamp (8 bytes). */ static class NullableFixedBinaryAsTimeStampReader extends NullableFixedByteAlignedReader<NullableTimeStampVector> { + /** + * The width of each element of the TimeStampVector is 8 byte(s). + */ + private static final int TIMESTAMP_LENGTH_IN_BITS = 64; + NullableFixedBinaryAsTimeStampReader(ParquetRecordReader parentReader, int allocateSize, ColumnDescriptor descriptor, ColumnChunkMetaData columnChunkMetaData, boolean fixedLength, NullableTimeStampVector v, SchemaElement schemaElement) throws ExecutionSetupException { super(parentReader, allocateSize, descriptor, columnChunkMetaData, fixedLength, v, schemaElement); @@ -132,6 +137,9 @@ public class NullableFixedByteAlignedReaders { valueVec.getMutator().setSafe(valuesReadInCurrentPass + i, getDateTimeValueFromBinary(binaryTimeStampValue)); } } + // The nanos precision is cut to millis. Therefore the length of single timestamp value is 8 bytes(s) + // instead of 12 byte(s). + dataTypeLengthInBits = TIMESTAMP_LENGTH_IN_BITS; } } http://git-wip-us.apache.org/repos/asf/drill/blob/08ca5e09/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java index ae0e699..362c943 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java @@ -25,6 +25,7 @@ import java.io.File; import java.io.FileWriter; import java.math.BigDecimal; import java.sql.Date; +import java.sql.Timestamp; import java.util.Arrays; import java.util.ArrayList; import java.util.Collection; @@ -963,5 +964,23 @@ public class TestParquetWriter extends BaseTestQuery { } } + @Test // DRILL-5097 + public void testInt96TimeStampValueWidth() throws Exception { + try { + testBuilder() + .ordered() + .sqlQuery("select c, d from cp.`parquet/data.snappy.parquet` where d = '2015-07-18 13:52:51'") + .optionSettingQueriesForTestQuery( + "alter session set `%s` = true", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP) + .baselineColumns("c", "d") + .baselineValues(new DateTime(Date.valueOf("2011-04-11").getTime()), + new DateTime(Timestamp.valueOf("2015-07-18 13:52:51").getTime())) + .build() + .run(); + } finally { + test("alter system reset `%s`", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP); + } + } + } http://git-wip-us.apache.org/repos/asf/drill/blob/08ca5e09/exec/java-exec/src/test/resources/parquet/data.snappy.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/data.snappy.parquet b/exec/java-exec/src/test/resources/parquet/data.snappy.parquet new file mode 100644 index 0000000..1ce3d75 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/data.snappy.parquet differ