http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java new file mode 100644 index 0000000..f9b0220 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/writer/TestCorruptParquetDateCorrection.java @@ -0,0 +1,539 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p/> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p/> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.physical.impl.writer; + +import org.apache.drill.PlanTestBase; +import org.apache.drill.TestBuilder; +import org.apache.drill.common.util.TestTools; +import org.apache.drill.exec.ExecConstants; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.joda.time.DateTime; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Tests for compatibility reading old parquet files after date corruption + * issue was fixed in DRILL-4203. + * + * Drill was writing non-standard dates into parquet files for all releases + * before 1.9.0. The values have been read by Drill correctly by Drill, but + * external tools like Spark reading the files will see corrupted values for + * all dates that have been written by Drill. + * + * This change corrects the behavior of the Drill parquet writer to correctly + * store dates in the format given in the parquet specification. + * + * To maintain compatibility with old files, the parquet reader code has + * been updated to check for the old format and automatically shift the + * corrupted values into corrected ones automatically. + * + * The test cases included here should ensure that all files produced by + * historical versions of Drill will continue to return the same values they + * had in previous releases. For compatibility with external tools, any old + * files with corrupted dates can be re-written using the CREATE TABLE AS + * command (as the writer will now only produce the specification-compliant + * values, even if after reading out of older corrupt files). + * + * While the old behavior was a consistent shift into an unlikely range + * to be used in a modern database (over 10,000 years in the future), these are still + * valid date values. In the case where these may have been written into + * files intentionally, and we cannot be certain from the metadata if Drill + * produced the files, an option is included to turn off the auto-correction. + * Use of this option is assumed to be extremely unlikely, but it is included + * for completeness. + */ +public class TestCorruptParquetDateCorrection extends PlanTestBase { + + // 4 files are in the directory: + // - one created with the fixed version of the reader, right before 1.9 + // - the code was changed to write the version number 1.9 (without snapshot) into the file + // - for compatibility all 1.9-SNAPSHOT files are read to correct the corrupt dates + // - one from and old version of Drill, before we put in proper created by in metadata + // - this is read properly by looking at a Max value in the file statistics, to see that + // it is way off of a typical date value + // - this behavior will be able to be turned off, but will be on by default + // - one from the 0.6 version of Drill, before files had min/max statistics + // - detecting corrupt values must be deferred to actual data page reading + // - one from 1.4, where there is a proper created-by, but the corruption is present + private static final String MIXED_CORRUPTED_AND_CORRECTED_DATES_PATH = + "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions"; + // partitioned with 1.4.0, date values are known to be corrupt + private static final String CORRUPTED_PARTITIONED_DATES_1_4_0_PATH = + "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203"; + // partitioned with 1.2.0, no certain metadata that these were written with Drill + // the value will be checked to see that they look corrupt and they will be corrected + // by default. Users can use the format plugin option autoCorrectCorruptDates to disable + // this behavior if they have foreign parquet files with valid rare date values that are + // in the similar range as Drill's corrupt values + private static final String CORRUPTED_PARTITIONED_DATES_1_2_PATH = + "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2"; + private static final String PARQUET_DATE_FILE_WITH_NULL_FILLED_COLS = + "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/null_date_cols_with_corruption_4203.parquet"; + private static final String CORRECTED_PARTITIONED_DATES_1_9_PATH = + "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption"; + private static final String VARCHAR_PARTITIONED = + "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition"; + private static final String DATE_PARTITIONED = + "[WORKING_PATH]/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition"; + + private static FileSystem fs; + private static Path path; + static String PARTITIONED_1_2_FOLDER = "partitioned_with_corruption_4203_1_2"; + static String MIXED_CORRUPTED_AND_CORRECTED_PARTITIONED_FOLDER = "mixed_partitioned"; + + @BeforeClass + public static void initFs() throws Exception { + Configuration conf = new Configuration(); + conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "local"); + fs = FileSystem.get(conf); + path = new Path(getDfsTestTmpSchemaLocation()); + + // Move files into temp directory, rewrite the metadata cache file to contain the appropriate absolute + // path + copyDirectoryIntoTempSpace(CORRUPTED_PARTITIONED_DATES_1_2_PATH); + copyMetaDataCacheToTempReplacingInternalPaths("parquet/4203_corrupt_dates/drill.parquet.metadata_1_2.requires_replace.txt", + PARTITIONED_1_2_FOLDER); + copyDirectoryIntoTempSpace(CORRUPTED_PARTITIONED_DATES_1_2_PATH, MIXED_CORRUPTED_AND_CORRECTED_PARTITIONED_FOLDER); + copyDirectoryIntoTempSpace(CORRECTED_PARTITIONED_DATES_1_9_PATH, MIXED_CORRUPTED_AND_CORRECTED_PARTITIONED_FOLDER); + copyDirectoryIntoTempSpace(CORRUPTED_PARTITIONED_DATES_1_4_0_PATH, MIXED_CORRUPTED_AND_CORRECTED_PARTITIONED_FOLDER); + } + + /** + * Test reading a directory full of partitioned parquet files with dates, these files have a drill version + * number of 1.9.0 in their footers, so we can be certain they do not have corruption. The option to disable the + * correction is passed, but it will not change the result in the case where we are certain correction + * is NOT needed. For more info see DRILL-4203. + */ + @Test + public void testReadPartitionedOnCorrectedDates() throws Exception { + try { + for (String selection : new String[]{"*", "date_col"}) { + // for sanity, try reading all partitions without a filter + TestBuilder builder = testBuilder() + .sqlQuery("select " + selection + " from table(dfs.`" + CORRECTED_PARTITIONED_DATES_1_9_PATH + "`" + + "(type => 'parquet', autoCorrectCorruptDates => false))") + .unOrdered() + .baselineColumns("date_col"); + addDateBaselineVals(builder); + builder.go(); + + String query = "select " + selection + " from table(dfs.`" + CORRECTED_PARTITIONED_DATES_1_9_PATH + "` " + + "(type => 'parquet', autoCorrectCorruptDates => false))" + " where date_col = date '1970-01-01'"; + // verify that pruning is actually taking place + testPlanMatchingPatterns(query, new String[]{"numFiles=1"}, null); + + // read with a filter on the partition column + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("date_col") + .baselineValues(new DateTime(1970, 1, 1, 0, 0)) + .go(); + } + } finally { + test("alter session reset all"); + } + } + + @Test + public void testVarcharPartitionedReadWithCorruption() throws Exception { + testBuilder() + .sqlQuery("select date_col from " + + "dfs.`" + VARCHAR_PARTITIONED + "`" + + "where length(varchar_col) = 12") + .baselineColumns("date_col") + .unOrdered() + .baselineValues(new DateTime(2039, 4, 9, 0, 0)) + .baselineValues(new DateTime(1999, 1, 8, 0, 0)) + .go(); + } + + @Test + public void testDatePartitionedReadWithCorruption() throws Exception { + testBuilder() + .sqlQuery("select date_col from " + + "dfs.`" + DATE_PARTITIONED + "`" + + "where date_col = '1999-04-08'") + .baselineColumns("date_col") + .unOrdered() + .baselineValues(new DateTime(1999, 4, 8, 0, 0)) + .go(); + + String sql = "select date_col from dfs.`" + DATE_PARTITIONED + "` where date_col > '1999-04-08'"; + testPlanMatchingPatterns(sql, new String[]{"numFiles=6"}, null); + } + + /** + * Test reading a directory full of partitioned parquet files with dates, these files have a drill version + * number of 1.4.0 in their footers, so we can be certain they are corrupt. The option to disable the + * correction is passed, but it will not change the result in the case where we are certain correction + * is needed. For more info see DRILL-4203. + */ + @Test + public void testReadPartitionedOnCorruptedDates() throws Exception { + try { + for (String selection : new String[]{"*", "date_col"}) { + // for sanity, try reading all partitions without a filter + TestBuilder builder = testBuilder() + .sqlQuery("select " + selection + " from table(dfs.`" + CORRUPTED_PARTITIONED_DATES_1_4_0_PATH + "`" + + "(type => 'parquet', autoCorrectCorruptDates => false))") + .unOrdered() + .baselineColumns("date_col"); + addDateBaselineVals(builder); + builder.go(); + + String query = "select " + selection + " from table(dfs.`" + CORRUPTED_PARTITIONED_DATES_1_4_0_PATH + "` " + + "(type => 'parquet', autoCorrectCorruptDates => false))" + " where date_col = date '1970-01-01'"; + // verify that pruning is actually taking place + testPlanMatchingPatterns(query, new String[]{"numFiles=1"}, null); + + // read with a filter on the partition column + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("date_col") + .baselineValues(new DateTime(1970, 1, 1, 0, 0)) + .go(); + } + } finally { + test("alter session reset all"); + } + } + + @Test + public void testReadPartitionedOnCorruptedDates_UserDisabledCorrection() throws Exception { + try { + for (String selection : new String[]{"*", "date_col"}) { + // for sanity, try reading all partitions without a filter + TestBuilder builder = testBuilder() + .sqlQuery("select " + selection + " from table(dfs.`" + CORRUPTED_PARTITIONED_DATES_1_2_PATH + "`" + + "(type => 'parquet', autoCorrectCorruptDates => false))") + .unOrdered() + .baselineColumns("date_col"); + addCorruptedDateBaselineVals(builder); + builder.go(); + + String query = "select " + selection + " from table(dfs.`" + CORRUPTED_PARTITIONED_DATES_1_2_PATH + "` " + + "(type => 'parquet', autoCorrectCorruptDates => false))" + " where date_col = cast('15334-03-17' as date)"; + // verify that pruning is actually taking place + testPlanMatchingPatterns(query, new String[]{"numFiles=1"}, null); + + // read with a filter on the partition column + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("date_col") + .baselineValues(new DateTime(15334, 03, 17, 0, 0)) + .go(); + } + } finally { + test("alter session reset all"); + } + } + + @Test + public void testCorruptValDetectionDuringPruning() throws Exception { + try { + for (String selection : new String[]{"*", "date_col"}) { + // for sanity, try reading all partitions without a filter + TestBuilder builder = testBuilder() + .sqlQuery("select " + selection + " from dfs.`" + CORRUPTED_PARTITIONED_DATES_1_2_PATH + "`") + .unOrdered() + .baselineColumns("date_col"); + addDateBaselineVals(builder); + builder.go(); + + String query = "select " + selection + " from dfs.`" + CORRUPTED_PARTITIONED_DATES_1_2_PATH + "`" + + " where date_col = date '1970-01-01'"; + // verify that pruning is actually taking place + testPlanMatchingPatterns(query, new String[]{"numFiles=1"}, null); + + // read with a filter on the partition column + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("date_col") + .baselineValues(new DateTime(1970, 1, 1, 0, 0)) + .go(); + } + } finally { + test("alter session reset all"); + } + } + + /** + * To fix some of the corrupted dates fixed as part of DRILL-4203 it requires + * actually looking at the values stored in the file. A column with date values + * actually stored must be located to check a value. Just because we find one + * column where the all values are null does not mean we can safely avoid reading + * date columns with auto-correction, although null values do not need fixing, + * other columns may contain actual corrupt date values. + * + * This test checks the case where the first columns in the file are all null filled + * and a later column must be found to identify that the file is corrupt. + */ + @Test + public void testReadCorruptDatesWithNullFilledColumns() throws Exception { + testBuilder() + .sqlQuery("select null_dates_1, null_dates_2, non_existent_field, date_col from dfs.`" + PARQUET_DATE_FILE_WITH_NULL_FILLED_COLS + "`") + .unOrdered() + .baselineColumns("null_dates_1", "null_dates_2", "non_existent_field", "date_col") + .baselineValues(null, null, null, new DateTime(1970, 1, 1, 0, 0)) + .baselineValues(null, null, null, new DateTime(1970, 1, 2, 0, 0)) + .baselineValues(null, null, null, new DateTime(1969, 12, 31, 0, 0)) + .baselineValues(null, null, null, new DateTime(1969, 12, 30, 0, 0)) + .baselineValues(null, null, null, new DateTime(1900, 1, 1, 0, 0)) + .baselineValues(null, null, null, new DateTime(2015, 1, 1, 0, 0)) + .go(); + } + + @Test + public void testUserOverrideDateCorrection() throws Exception { + // read once with the flat reader + readFilesWithUserDisabledAutoCorrection(); + + try { + test(String.format("alter session set %s = true", ExecConstants.PARQUET_NEW_RECORD_READER)); + // read all of the types with the complex reader + readFilesWithUserDisabledAutoCorrection(); + } finally { + test("alter session reset all"); + } + + } + + /** + * Test reading a directory full of parquet files with dates, some of which have corrupted values + * due to DRILL-4203. + * + * Tests reading the files with both the vectorized and complex parquet readers. + * + * @throws Exception + */ + @Test + public void testReadMixedOldAndNewBothReaders() throws Exception { + /// read once with the flat reader + readMixedCorruptedAndCorrectedDates(); + + try { + // read all of the types with the complex reader + test(String.format("alter session set %s = true", ExecConstants.PARQUET_NEW_RECORD_READER)); + readMixedCorruptedAndCorrectedDates(); + } finally { + test(String.format("alter session set %s = false", ExecConstants.PARQUET_NEW_RECORD_READER)); + } + } + + public void addDateBaselineVals(TestBuilder builder) { + builder + .baselineValues(new DateTime(1970, 1, 1, 0, 0)) + .baselineValues(new DateTime(1970, 1, 2, 0, 0)) + .baselineValues(new DateTime(1969, 12, 31, 0, 0)) + .baselineValues(new DateTime(1969, 12, 30, 0, 0)) + .baselineValues(new DateTime(1900, 1, 1, 0, 0)) + .baselineValues(new DateTime(2015, 1, 1, 0, 0)); + } + + /** + * These are the same values added in the addDateBaselineVals, shifted as corrupt values + */ + public void addCorruptedDateBaselineVals(TestBuilder builder) { + builder + .baselineValues(new DateTime(15334, 03, 17, 0, 0)) + .baselineValues(new DateTime(15334, 03, 18, 0, 0)) + .baselineValues(new DateTime(15334, 03, 15, 0, 0)) + .baselineValues(new DateTime(15334, 03, 16, 0, 0)) + .baselineValues(new DateTime(15264, 03, 16, 0, 0)) + .baselineValues(new DateTime(15379, 03, 17, 0, 0)); + } + + public void readFilesWithUserDisabledAutoCorrection() throws Exception { + // ensure that selecting the date column explicitly or as part of a star still results + // in checking the file metadata for date columns (when we need to check the statistics + // for bad values) to set the flag that the values are corrupt + for (String selection : new String[] {"*", "date_col"}) { + TestBuilder builder = testBuilder() + .sqlQuery("select " + selection + " from table(dfs.`" + MIXED_CORRUPTED_AND_CORRECTED_DATES_PATH + "`" + + "(type => 'parquet', autoCorrectCorruptDates => false))") + .unOrdered() + .baselineColumns("date_col"); + addDateBaselineVals(builder); + addDateBaselineVals(builder); + addCorruptedDateBaselineVals(builder); + addCorruptedDateBaselineVals(builder); + builder.go(); + } + } + + private static String replaceWorkingPathInString(String orig) { + return orig.replaceAll(Pattern.quote("[WORKING_PATH]"), Matcher.quoteReplacement(TestTools.getWorkingPath())); + } + + private static void copyDirectoryIntoTempSpace(String resourcesDir) throws IOException { + copyDirectoryIntoTempSpace(resourcesDir, null); + } + + private static void copyDirectoryIntoTempSpace(String resourcesDir, String destinationSubDir) throws IOException { + Path destination = path; + if (destinationSubDir != null) { + destination = new Path(path, destinationSubDir); + } + fs.copyFromLocalFile( + new Path(replaceWorkingPathInString(resourcesDir)), + destination); + } + + /** + * Metadata cache files include full paths to the files that have been scanned. + * + * There is no way to generate a metadata cache file with absolute paths that + * will be guarenteed to be available on an arbitrary test machine. + * + * To enable testing older metadata cache files, they were generated manually + * using older drill versions, and the absolute path up to the folder where + * the metadata cache file appeared was manually replaced with the string + * REPLACED_IN_TEST. Here the file is re-written into the given temporary + * location after the REPLACED_IN_TEST string has been replaced by the actual + * location generated during this run of the tests. + * + * @param srcFileOnClassPath + * @param destFolderInTmp + * @throws IOException + */ + private static void copyMetaDataCacheToTempReplacingInternalPaths(String srcFileOnClassPath, String destFolderInTmp) throws IOException { + String metadataFileContents = getFile(srcFileOnClassPath); + Path newMetaCache = new Path(new Path(path, destFolderInTmp), ".drill.parquet_metadata"); + FSDataOutputStream outSteam = fs.create(newMetaCache); + outSteam.writeBytes(metadataFileContents.replace("REPLACED_IN_TEST", path.toString())); + outSteam.close(); + } + + @Test + public void testReadOldMetadataCacheFile() throws Exception { + // for sanity, try reading all partitions without a filter + String query = "select date_col from dfs.`" + new Path(path, PARTITIONED_1_2_FOLDER) + "`"; + TestBuilder builder = testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("date_col"); + addDateBaselineVals(builder); + builder.go(); + testPlanMatchingPatterns(query, new String[]{"usedMetadataFile=true"}, null); + } + + @Test + public void testReadOldMetadataCacheFileWithPruning() throws Exception { + String query = "select date_col from dfs.`" + new Path(path, PARTITIONED_1_2_FOLDER) + "`" + + " where date_col = date '1970-01-01'"; + // verify that pruning is actually taking place + testPlanMatchingPatterns(query, new String[]{"numFiles=1", "usedMetadataFile=true"}, null); + + // read with a filter on the partition column + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("date_col") + .baselineValues(new DateTime(1970, 1, 1, 0, 0)) + .go(); + } + + @Test + public void testReadOldMetadataCacheFileOverrideCorrection() throws Exception { + // for sanity, try reading all partitions without a filter + TestBuilder builder = testBuilder() + .sqlQuery("select date_col from table(dfs.`" + new Path(path, PARTITIONED_1_2_FOLDER) + "`" + + "(type => 'parquet', autoCorrectCorruptDates => false))") + .unOrdered() + .baselineColumns("date_col"); + addCorruptedDateBaselineVals(builder); + builder.go(); + + String query = "select date_col from table(dfs.`" + new Path(path, PARTITIONED_1_2_FOLDER) + "` " + + "(type => 'parquet', autoCorrectCorruptDates => false))" + " where date_col = cast('15334-03-17' as date)"; + // verify that pruning is actually taking place + testPlanMatchingPatterns(query, new String[]{"numFiles=1", "usedMetadataFile=true"}, null); + + // read with a filter on the partition column + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("date_col") + .baselineValues(new DateTime(15334, 03, 17, 0, 0)) + .go(); + } + + @Test + public void testReadNewMetadataCacheFileOverOldAndNewFiles() throws Exception { + String table = "dfs.`" + new Path(path, MIXED_CORRUPTED_AND_CORRECTED_PARTITIONED_FOLDER) + "`"; + copyMetaDataCacheToTempReplacingInternalPaths("parquet/4203_corrupt_dates/mixed_version_partitioned_metadata.requires_replace.txt", + MIXED_CORRUPTED_AND_CORRECTED_PARTITIONED_FOLDER); + // for sanity, try reading all partitions without a filter + TestBuilder builder = testBuilder() + .sqlQuery("select date_col from " + table) + .unOrdered() + .baselineColumns("date_col"); + addDateBaselineVals(builder); + addDateBaselineVals(builder); + addDateBaselineVals(builder); + builder.go(); + + String query = "select date_col from " + table + + " where date_col = date '1970-01-01'"; + // verify that pruning is actually taking place + testPlanMatchingPatterns(query, new String[]{"numFiles=3", "usedMetadataFile=true"}, null); + + // read with a filter on the partition column + testBuilder() + .sqlQuery(query) + .unOrdered() + .baselineColumns("date_col") + .baselineValues(new DateTime(1970, 1, 1, 0, 0)) + .baselineValues(new DateTime(1970, 1, 1, 0, 0)) + .baselineValues(new DateTime(1970, 1, 1, 0, 0)) + .go(); + } + + /** + * Read a directory with parquet files where some have corrupted dates, see DRILL-4203. + * @throws Exception + */ + public void readMixedCorruptedAndCorrectedDates() throws Exception { + // ensure that selecting the date column explicitly or as part of a star still results + // in checking the file metadata for date columns (when we need to check the statistics + // for bad values) to set the flag that the values are corrupt + for (String selection : new String[] {"*", "date_col"}) { + TestBuilder builder = testBuilder() + .sqlQuery("select " + selection + " from dfs.`" + MIXED_CORRUPTED_AND_CORRECTED_DATES_PATH + "`") + .unOrdered() + .baselineColumns("date_col"); + for (int i = 0; i < 4; i++) { + addDateBaselineVals(builder); + } + builder.go(); + } + } + +}
http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java index cdeafae..c341295 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/dfs/TestFormatPluginOptionExtractor.java @@ -53,9 +53,11 @@ public class TestFormatPluginOptionExtractor { assertEquals(NamedFormatPluginConfig.class, d.pluginConfigClass); assertEquals("(type: String, name: String)", d.presentParams()); break; + case "parquet": + assertEquals(d.typeName, "(type: String, autoCorrectCorruptDates: boolean)", d.presentParams()); + break; case "json": case "sequencefile": - case "parquet": case "avro": assertEquals(d.typeName, "(type: String)", d.presentParams()); break; http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java index 26ba316..51fa45c 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/parquet/ParquetRecordReaderTest.java @@ -639,7 +639,7 @@ public class ParquetRecordReaderTest extends BaseTestQuery { for(int i = 0; i < 25; i++) { final ParquetRecordReader rr = new ParquetRecordReader(context, 256000, fileName, 0, fs, CodecFactory.createDirectCodecFactory(dfsConfig, new ParquetDirectByteBufferAllocator(allocator), 0), - f.getParquetMetadata(), columns); + f.getParquetMetadata(), columns, ParquetReaderUtility.DateCorruptionStatus.META_SHOWS_CORRUPTION); final TestOutputMutator mutator = new TestOutputMutator(allocator); rr.setup(null, mutator); final Stopwatch watch = Stopwatch.createStarted(); http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_1.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_1.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_1.parquet new file mode 100644 index 0000000..7aa8e61 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_1.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_2.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_2.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_2.parquet new file mode 100644 index 0000000..8c43cee Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_2.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_3.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_3.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_3.parquet new file mode 100644 index 0000000..2d2415a Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_3.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_4.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_4.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_4.parquet new file mode 100644 index 0000000..ff5ce24 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_4.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_5.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_5.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_5.parquet new file mode 100644 index 0000000..4a5d4fb Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_5.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_6.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_6.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_6.parquet new file mode 100644 index 0000000..da7ad3f Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/1_9_0_partitioned_no_corruption/0_0_6.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/drill.parquet.metadata_1_2.requires_replace.txt ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/drill.parquet.metadata_1_2.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/drill.parquet.metadata_1_2.requires_replace.txt new file mode 100644 index 0000000..bfca095 --- /dev/null +++ b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/drill.parquet.metadata_1_2.requires_replace.txt @@ -0,0 +1,119 @@ +{ + "metadata_version" : "v1", + "files" : [ { + "path" : "REPLACED_IN_TEST/partitioned_with_corruption_4203_1_2/0_0_1.parquet", + "length" : 160, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : "`date_col`", + "primitiveType" : "INT32", + "originalType" : "DATE", + "max" : 4855609, + "min" : 4855609, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/partitioned_with_corruption_4203_1_2/0_0_2.parquet", + "length" : 160, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : "`date_col`", + "primitiveType" : "INT32", + "originalType" : "DATE", + "max" : 4881174, + "min" : 4881174, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/partitioned_with_corruption_4203_1_2/0_0_3.parquet", + "length" : 160, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : "`date_col`", + "primitiveType" : "INT32", + "originalType" : "DATE", + "max" : 4881175, + "min" : 4881175, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/partitioned_with_corruption_4203_1_2/0_0_4.parquet", + "length" : 160, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : "`date_col`", + "primitiveType" : "INT32", + "originalType" : "DATE", + "max" : 4881176, + "min" : 4881176, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/partitioned_with_corruption_4203_1_2/0_0_5.parquet", + "length" : 160, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : "`date_col`", + "primitiveType" : "INT32", + "originalType" : "DATE", + "max" : 4881177, + "min" : 4881177, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/partitioned_with_corruption_4203_1_2/0_0_6.parquet", + "length" : 160, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : "`date_col`", + "primitiveType" : "INT32", + "originalType" : "DATE", + "max" : 4897612, + "min" : 4897612, + "nulls" : 0 + } ] + } ] + } ], + "directories" : [ ] +} http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_1.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_1.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_1.parquet new file mode 100644 index 0000000..9890a05 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_1.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_10.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_10.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_10.parquet new file mode 100644 index 0000000..c956e68 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_10.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_11.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_11.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_11.parquet new file mode 100644 index 0000000..29839d2 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_11.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_12.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_12.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_12.parquet new file mode 100644 index 0000000..af0261c Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_12.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_13.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_13.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_13.parquet new file mode 100644 index 0000000..67eff64 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_13.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_14.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_14.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_14.parquet new file mode 100644 index 0000000..7be1e79 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_14.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_15.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_15.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_15.parquet new file mode 100644 index 0000000..89be46d Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_15.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_16.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_16.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_16.parquet new file mode 100644 index 0000000..0a00b2e Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_16.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_17.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_17.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_17.parquet new file mode 100644 index 0000000..ed37e62 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_17.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_18.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_18.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_18.parquet new file mode 100644 index 0000000..d769ecc Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_18.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_19.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_19.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_19.parquet new file mode 100644 index 0000000..ce72a9c Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_19.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_2.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_2.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_2.parquet new file mode 100644 index 0000000..3b967e5 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_2.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_20.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_20.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_20.parquet new file mode 100644 index 0000000..6402af6 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_20.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_21.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_21.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_21.parquet new file mode 100644 index 0000000..c1ac138 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_21.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_3.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_3.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_3.parquet new file mode 100644 index 0000000..fbed0cf Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_3.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_4.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_4.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_4.parquet new file mode 100644 index 0000000..33a1989 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_4.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_5.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_5.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_5.parquet new file mode 100644 index 0000000..cdfadf0 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_5.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_6.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_6.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_6.parquet new file mode 100644 index 0000000..cfffda8 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_6.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_7.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_7.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_7.parquet new file mode 100644 index 0000000..55ac2cf Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_7.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_8.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_8.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_8.parquet new file mode 100644 index 0000000..1ba9706 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_8.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_9.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_9.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_9.parquet new file mode 100644 index 0000000..a1a27fa Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_datepartition/0_0_9.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_1.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_1.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_1.parquet new file mode 100644 index 0000000..01d9603 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_1.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_10.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_10.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_10.parquet new file mode 100644 index 0000000..e3d0e3f Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_10.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_11.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_11.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_11.parquet new file mode 100644 index 0000000..2311590 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_11.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_12.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_12.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_12.parquet new file mode 100644 index 0000000..65f494c Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_12.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_13.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_13.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_13.parquet new file mode 100644 index 0000000..55b39fd Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_13.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_14.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_14.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_14.parquet new file mode 100644 index 0000000..bbd1227 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_14.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_15.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_15.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_15.parquet new file mode 100644 index 0000000..469e223 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_15.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_16.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_16.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_16.parquet new file mode 100644 index 0000000..46a7b59 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_16.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_17.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_17.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_17.parquet new file mode 100644 index 0000000..25a2b20 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_17.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_18.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_18.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_18.parquet new file mode 100644 index 0000000..8df4d05 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_18.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_19.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_19.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_19.parquet new file mode 100644 index 0000000..dbc1d9f Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_19.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_2.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_2.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_2.parquet new file mode 100644 index 0000000..6819f63 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_2.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_20.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_20.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_20.parquet new file mode 100644 index 0000000..ad5967a Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_20.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_21.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_21.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_21.parquet new file mode 100644 index 0000000..277d681 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_21.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_3.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_3.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_3.parquet new file mode 100644 index 0000000..9d1f5f3 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_3.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_4.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_4.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_4.parquet new file mode 100644 index 0000000..e3e265a Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_4.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_5.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_5.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_5.parquet new file mode 100644 index 0000000..e2fd411 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_5.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_6.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_6.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_6.parquet new file mode 100644 index 0000000..a9df633 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_6.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_7.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_7.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_7.parquet new file mode 100644 index 0000000..ebe8183 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_7.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_8.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_8.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_8.parquet new file mode 100644 index 0000000..c5c6793 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_8.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_9.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_9.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_9.parquet new file mode 100644 index 0000000..d32fc3f Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/fewtypes_varcharpartition/0_0_9.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrected_dates.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrected_dates.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrected_dates.parquet new file mode 100644 index 0000000..cee02a2 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrected_dates.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrupt_dates.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrupt_dates.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrupt_dates.parquet new file mode 100644 index 0000000..64f7568 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrupt_dates.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrupted_dates_1.4.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrupted_dates_1.4.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrupted_dates_1.4.parquet new file mode 100644 index 0000000..62429fe Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/4203_corrupted_dates_1.4.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/drill_0_6_currupt_dates_no_stats.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/drill_0_6_currupt_dates_no_stats.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/drill_0_6_currupt_dates_no_stats.parquet new file mode 100644 index 0000000..984074e Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_drill_versions/drill_0_6_currupt_dates_no_stats.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_version_partitioned_metadata.requires_replace.txt ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_version_partitioned_metadata.requires_replace.txt b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_version_partitioned_metadata.requires_replace.txt new file mode 100644 index 0000000..7fdb5b2 --- /dev/null +++ b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/mixed_version_partitioned_metadata.requires_replace.txt @@ -0,0 +1,301 @@ +{ + "metadata_version" : "v2", + "columnTypeInfo" : { + "date_col" : { + "name" : [ "date_col" ], + "primitiveType" : "INT32", + "originalType" : "DATE" + } + }, + "files" : [ { + "path" : "REPLACED_IN_TEST/mixed_partitioned/1_9_0_partitioned_no_corruption/0_0_1.parquet", + "length" : 257, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : -25567, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/1_9_0_partitioned_no_corruption/0_0_2.parquet", + "length" : 257, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : -2, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/1_9_0_partitioned_no_corruption/0_0_3.parquet", + "length" : 257, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : -1, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/1_9_0_partitioned_no_corruption/0_0_4.parquet", + "length" : 257, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : 0, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/1_9_0_partitioned_no_corruption/0_0_5.parquet", + "length" : 257, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : 1, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/1_9_0_partitioned_no_corruption/0_0_6.parquet", + "length" : 257, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : 16436, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/partitioned_with_corruption_4203/0_0_1.parquet", + "length" : 257, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : -25567, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/partitioned_with_corruption_4203/0_0_2.parquet", + "length" : 257, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : -2, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/partitioned_with_corruption_4203/0_0_3.parquet", + "length" : 257, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : -1, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/partitioned_with_corruption_4203/0_0_4.parquet", + "length" : 257, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : 0, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/partitioned_with_corruption_4203/0_0_5.parquet", + "length" : 257, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : 1, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/partitioned_with_corruption_4203/0_0_6.parquet", + "length" : 257, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : 16436, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/0_0_1.parquet", + "length" : 160, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : -25567, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/0_0_2.parquet", + "length" : 160, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : -2, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/0_0_3.parquet", + "length" : 160, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : -1, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/0_0_4.parquet", + "length" : 160, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : 0, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/0_0_5.parquet", + "length" : 160, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : 1, + "nulls" : 0 + } ] + } ] + }, { + "path" : "REPLACED_IN_TEST/mixed_partitioned/0_0_6.parquet", + "length" : 160, + "rowGroups" : [ { + "start" : 4, + "length" : 45, + "rowCount" : 1, + "hostAffinity" : { + "localhost" : 1.0 + }, + "columns" : [ { + "name" : [ "date_col" ], + "mxValue" : 16436, + "nulls" : 0 + } ] + } ] + } ], + "directories" : [ "file:REPLACED_IN_TEST/mixed_partitioned/1_9_0_partitioned_no_corruption", "file:REPLACED_IN_TEST/mixed_partitioned/partitioned_with_corruption_4203" ], + "drillVersion" : "1.9.0" +} http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/null_date_cols_with_corruption_4203.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/null_date_cols_with_corruption_4203.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/null_date_cols_with_corruption_4203.parquet new file mode 100644 index 0000000..c5c0b1a Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/null_date_cols_with_corruption_4203.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_1.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_1.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_1.parquet new file mode 100644 index 0000000..31723cc Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_1.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_2.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_2.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_2.parquet new file mode 100644 index 0000000..0c558ed Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_2.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_3.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_3.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_3.parquet new file mode 100644 index 0000000..f069ddf Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_3.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_4.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_4.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_4.parquet new file mode 100644 index 0000000..2c0dd7b Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_4.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_5.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_5.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_5.parquet new file mode 100644 index 0000000..19a436b Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_5.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_6.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_6.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_6.parquet new file mode 100644 index 0000000..49020b5 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203/0_0_6.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_1.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_1.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_1.parquet new file mode 100644 index 0000000..ba99a37 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_1.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_2.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_2.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_2.parquet new file mode 100644 index 0000000..7e1442b Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_2.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_3.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_3.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_3.parquet new file mode 100644 index 0000000..ac3f88e Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_3.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_4.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_4.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_4.parquet new file mode 100644 index 0000000..c6550f8 Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_4.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_5.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_5.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_5.parquet new file mode 100644 index 0000000..fef0eac Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_5.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/ae34d5c3/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_6.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_6.parquet b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_6.parquet new file mode 100644 index 0000000..22df04a Binary files /dev/null and b/exec/java-exec/src/test/resources/parquet/4203_corrupt_dates/partitioned_with_corruption_4203_1_2/0_0_6.parquet differ
