Github user vdiravka commented on a diff in the pull request: https://github.com/apache/drill/pull/600#discussion_r85522267 --- Diff: exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestParquetWriter.java --- @@ -739,30 +741,76 @@ public void runTestAndValidate(String selection, String validationSelection, Str } /* - Test the reading of an int96 field. Impala encodes timestamps as int96 fields + Impala encodes timestamp values as int96 fields. Test the reading of an int96 field with two converters: + the first one converts parquet INT96 into drill VARBINARY and the second one (works while + store.parquet.reader.int96_as_timestamp option is enabled) converts parquet INT96 into drill TIMESTAMP. */ @Test public void testImpalaParquetInt96() throws Exception { compareParquetReadersColumnar("field_impala_ts", "cp.`parquet/int96_impala_1.parquet`"); + try { + test("alter session set %s = true", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP); + compareParquetReadersColumnar("field_impala_ts", "cp.`parquet/int96_impala_1.parquet`"); + } finally { + test("alter session reset %s", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP); + } } /* - Test the reading of a binary field where data is in dicationary _and_ non-dictionary encoded pages + Test the reading of a binary field as drill varbinary where data is in dicationary _and_ non-dictionary encoded pages */ @Test - public void testImpalaParquetVarBinary_DictChange() throws Exception { + public void testImpalaParquetBinaryAsVarBinary_DictChange() throws Exception { compareParquetReadersColumnar("field_impala_ts", "cp.`parquet/int96_dict_change.parquet`"); } /* + Test the reading of a binary field as drill timestamp where data is in dicationary _and_ non-dictionary encoded pages + */ + @Test + public void testImpalaParquetBinaryAsTimeStamp_DictChange() throws Exception { + final String WORKING_PATH = TestTools.getWorkingPath(); + final String TEST_RES_PATH = WORKING_PATH + "/src/test/resources"; + try { + testBuilder() + .sqlQuery("select int96_ts from dfs_test.`%s/parquet/int96_dict_change`", TEST_RES_PATH) + .optionSettingQueriesForTestQuery( + "alter session set `%s` = true", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP) + .ordered() + .csvBaselineFile("testframework/testParquetReader/testInt96DictChange/q1.tsv") + .baselineTypes(TypeProtos.MinorType.TIMESTAMP) + .baselineColumns("int96_ts") + .build().run(); + } finally { + test("alter system reset `%s`", ExecConstants.PARQUET_READER_INT96_AS_TIMESTAMP); + } + } + + /* Test the conversion from int96 to impala timestamp */ @Test - public void testImpalaParquetTimestampAsInt96() throws Exception { + public void testTimestampImpalaConvertFrom() throws Exception { compareParquetReadersColumnar("convert_from(field_impala_ts, 'TIMESTAMP_IMPALA')", "cp.`parquet/int96_impala_1.parquet`"); } /* + Test reading parquet Int96 as TimeStamp and comparing obtained values with the + old results (reading the same values as VarBinary and convert_fromTIMESTAMP_IMPALA function using) + */ + @Test + public void testImpalaParquetTimestampInt96AsTimeStamp() throws Exception { --- End diff -- This test compares the results between new converter (Int96 to TimeStamp) and the old one (Int96 to VarBinary) with `convert_fromTIMESTAMP_IMPALA` function. The issue was in the `ConvertFromImpalaTimestamp` [link to the code ](https://github.com/apache/drill/pull/600/commits/a45490af2dd663168220cc3bda62a2d79170db62#diff-5d8360c5e3cf7d2f6ac7bfe58b6d319aL57) Because the timezone changing shouldn't affect on the result timestamp values. I deleted timezone consideration there, so now all tests passed successfuly even across different timezones.
--- If your project is set up for it, you can reply to this email and have your reply appear on GitHub as well. If your project does not have this feature enabled and wishes so, or if the feature is enabled but not working, please contact infrastructure at infrastruct...@apache.org or file a JIRA ticket with INFRA. ---