Repository: drill Updated Branches: refs/heads/master fbdd20e54 -> e047e1089
DRILL-4733: max(dir0) reading more columns than necessary Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/e047e108 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/e047e108 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/e047e108 Branch: refs/heads/master Commit: e047e1089f84a79eeab07bd13b8a5ca46e84b7d3 Parents: fbdd20e Author: Arina Ielchiieva <[email protected]> Authored: Tue Jun 21 12:33:32 2016 +0000 Committer: Aman Sinha <[email protected]> Committed: Thu Jun 23 08:03:11 2016 -0700 ---------------------------------------------------------------------- .../drill/exec/store/ImplicitColumnExplorer.java | 6 ------ .../exec/store/dfs/easy/EasyFormatPlugin.java | 7 ++++++- .../drill/exec/store/TestImplicitFileColumns.java | 17 +++++++++++++++++ .../parquetWithSchemaChange/voter5/voter5.parquet | Bin 0 -> 1250 bytes .../voter50/voter50.parquet | Bin 0 -> 2960 bytes 5 files changed, 23 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/e047e108/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java index 94a0dca..af74eb7 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/ImplicitColumnExplorer.java @@ -135,12 +135,6 @@ public class ImplicitColumnExplorer { tableColumns.add(column); } } - - // We must make sure to pass a table column(not to be confused with partition column) to the underlying record - // reader. - if (tableColumns.size() == 0) { - tableColumns.add(AbstractRecordReader.STAR_COLUMN); - } } } http://git-wip-us.apache.org/repos/asf/drill/blob/e047e108/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java index 5881d33..f56f445 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyFormatPlugin.java @@ -41,6 +41,7 @@ import org.apache.drill.exec.planner.physical.PlannerSettings; import org.apache.drill.exec.record.CloseableRecordBatch; import org.apache.drill.exec.record.RecordBatch; import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.store.AbstractRecordReader; import org.apache.drill.exec.store.ImplicitColumnExplorer; import org.apache.drill.exec.store.RecordReader; import org.apache.drill.exec.store.RecordWriter; @@ -126,8 +127,12 @@ public abstract class EasyFormatPlugin<T extends FormatPluginConfig> implements final ImplicitColumnExplorer columnExplorer = new ImplicitColumnExplorer(context, scan.getColumns()); if (!columnExplorer.isSelectAllColumns()) { + // We must make sure to pass a table column (not to be confused with implicit column) to the underlying record reader. + List<SchemaPath> tableColumns = + columnExplorer.getTableColumns().size() == 0 ? + Lists.<SchemaPath>newArrayList(AbstractRecordReader.STAR_COLUMN) : columnExplorer.getTableColumns(); scan = new EasySubScan(scan.getUserName(), scan.getWorkUnits(), scan.getFormatPlugin(), - columnExplorer.getTableColumns(), scan.getSelectionRoot()); + tableColumns, scan.getSelectionRoot()); scan.setOperatorId(scan.getOperatorId()); } http://git-wip-us.apache.org/repos/asf/drill/blob/e047e108/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java index 6900da9..ce0f5e9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/TestImplicitFileColumns.java @@ -20,6 +20,7 @@ package org.apache.drill.exec.store; import com.google.common.base.Charsets; import com.google.common.io.Files; import org.apache.drill.BaseTestQuery; +import org.apache.drill.common.util.TestTools; import org.apache.drill.exec.util.JsonStringArrayList; import org.apache.drill.exec.util.Text; import org.apache.hadoop.fs.Path; @@ -110,4 +111,20 @@ public class TestImplicitFileColumns extends BaseTestQuery { .go(); } + @Test // DRILL-4733 + public void testMultilevelParquetWithSchemaChange() throws Exception { + try { + test("alter session set `planner.enable_decimal_data_type` = true"); + testBuilder() + .sqlQuery(String.format("select max(dir0) as max_dir from dfs_test.`%s/src/test/resources/multilevel/parquetWithSchemaChange`", + TestTools.getWorkingPath())) + .unOrdered() + .baselineColumns("max_dir") + .baselineValues("voter50") + .go(); + } finally { + test("alter session set `planner.enable_decimal_data_type` = false"); + } + } + } http://git-wip-us.apache.org/repos/asf/drill/blob/e047e108/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter5/voter5.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter5/voter5.parquet b/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter5/voter5.parquet new file mode 100644 index 0000000..cc76280 Binary files /dev/null and b/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter5/voter5.parquet differ http://git-wip-us.apache.org/repos/asf/drill/blob/e047e108/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter50/voter50.parquet ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter50/voter50.parquet b/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter50/voter50.parquet new file mode 100644 index 0000000..ff66b42 Binary files /dev/null and b/exec/java-exec/src/test/resources/multilevel/parquetWithSchemaChange/voter50/voter50.parquet differ
