Repository: drill Updated Branches: refs/heads/master a45f7fd11 -> c67d070bf
DRILL-4387: GroupScan or ScanBatchCreator should not use star column in case of skipAll query. The skipAll query should be handled in RecordReader. Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/c67d070b Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/c67d070b Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/c67d070b Branch: refs/heads/master Commit: c67d070bf68e9492b7fbe6d9e54d70d587b28a6e Parents: a45f7fd Author: Jinfeng Ni <[email protected]> Authored: Fri Feb 12 14:18:59 2016 -0800 Committer: Jinfeng Ni <[email protected]> Committed: Mon Feb 22 11:54:11 2016 -0800 ---------------------------------------------------------------------- .../org/apache/drill/exec/store/hbase/HBaseGroupScan.java | 6 +++--- .../java/org/apache/drill/exec/physical/base/GroupScan.java | 6 ++++++ .../org/apache/drill/exec/store/AbstractRecordReader.java | 7 +++++++ .../org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java | 2 +- .../apache/drill/exec/store/parquet/ParquetRowGroupScan.java | 2 +- .../drill/exec/store/parquet/ParquetScanBatchCreator.java | 3 --- 6 files changed, 18 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/c67d070b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java ---------------------------------------------------------------------- diff --git a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java index 1d01635..ae243b3 100644 --- a/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java +++ b/contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/HBaseGroupScan.java @@ -114,7 +114,7 @@ public class HBaseGroupScan extends AbstractGroupScan implements DrillHBaseConst this.storagePlugin = storagePlugin; this.storagePluginConfig = storagePlugin.getConfig(); this.hbaseScanSpec = scanSpec; - this.columns = columns == null || columns.size() == 0? ALL_COLUMNS : columns; + this.columns = columns == null ? ALL_COLUMNS : columns; init(); } @@ -124,7 +124,7 @@ public class HBaseGroupScan extends AbstractGroupScan implements DrillHBaseConst */ private HBaseGroupScan(HBaseGroupScan that) { super(that); - this.columns = that.columns; + this.columns = that.columns == null ? ALL_COLUMNS : that.columns; this.hbaseScanSpec = that.hbaseScanSpec; this.endpointFragmentMapping = that.endpointFragmentMapping; this.regionsToScan = that.regionsToScan; @@ -139,7 +139,7 @@ public class HBaseGroupScan extends AbstractGroupScan implements DrillHBaseConst @Override public GroupScan clone(List<SchemaPath> columns) { HBaseGroupScan newScan = new HBaseGroupScan(this); - newScan.columns = columns; + newScan.columns = columns == null ? ALL_COLUMNS : columns;; newScan.verifyColumns(); return newScan; } http://git-wip-us.apache.org/repos/asf/drill/blob/c67d070b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java index 98acb0a..daaa5ac 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/base/GroupScan.java @@ -35,7 +35,13 @@ import com.google.common.collect.ImmutableList; */ public interface GroupScan extends Scan, HasAffinity{ + /** + * columns list in GroupScan : 1) empty_column is for skipAll query. + * 2) NULL is interpreted as ALL_COLUMNS. + * How to handle skipAll query is up to each storage plugin, with different policy in corresponding RecordReader. + */ public static final List<SchemaPath> ALL_COLUMNS = ImmutableList.of(SchemaPath.getSimplePath("*")); + public static final long NO_COLUMN_STATS = -1; public abstract void applyAssignments(List<DrillbitEndpoint> endpoints) throws PhysicalOperatorSetupException; http://git-wip-us.apache.org/repos/asf/drill/blob/c67d070b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java index 2ddcbaa..16118d9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/AbstractRecordReader.java @@ -54,6 +54,13 @@ public abstract class AbstractRecordReader implements RecordReader { + ", isSkipQuery = " + isSkipQuery + "]"; } + /** + * + * @param projected : The column list to be returned from this RecordReader. + * 1) empty column list: this is for skipAll query. It's up to each storage-plugin to + * choose different policy of handling skipAll query. By default, it will use * column. + * 2) NULL : is NOT allowed. It requires the planner's rule, or GroupScan or ScanBatchCreator to handle NULL. + */ protected final void setColumns(Collection<SchemaPath> projected) { Preconditions.checkNotNull(projected, COL_NULL_ERROR); isSkipQuery = projected.isEmpty(); http://git-wip-us.apache.org/repos/asf/drill/blob/c67d070b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java index 791e4f7..ebea2f4 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java @@ -99,7 +99,7 @@ public class EasyGroupScan extends AbstractFileGroupScan{ super(userName); this.selection = Preconditions.checkNotNull(selection); this.formatPlugin = Preconditions.checkNotNull(formatPlugin, "Unable to load format plugin for provided format config."); - this.columns = columns == null || columns.size() == 0? ALL_COLUMNS : columns; + this.columns = columns == null ? ALL_COLUMNS : columns; this.selectionRoot = selectionRoot; initFromSelection(selection, formatPlugin); } http://git-wip-us.apache.org/repos/asf/drill/blob/c67d070b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java index 987f792..cffcdac 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetRowGroupScan.java @@ -78,7 +78,7 @@ public class ParquetRowGroupScan extends AbstractBase implements SubScan { this.formatPlugin = Preconditions.checkNotNull(formatPlugin); this.formatConfig = formatPlugin.getConfig(); this.rowGroupReadEntries = rowGroupReadEntries; - this.columns = columns == null || columns.size() == 0 ? GroupScan.ALL_COLUMNS : columns; + this.columns = columns == null ? GroupScan.ALL_COLUMNS : columns; this.selectionRoot = selectionRoot; } http://git-wip-us.apache.org/repos/asf/drill/blob/c67d070b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java index 485f84a..c730bc9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java @@ -87,9 +87,6 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan newColumns.add(column); } } - if (newColumns.isEmpty()) { - newColumns = GroupScan.ALL_COLUMNS; - } final int id = rowGroupScan.getOperatorId(); // Create the new row group scan with the new columns rowGroupScan = new ParquetRowGroupScan(rowGroupScan.getUserName(), rowGroupScan.getStorageEngine(),
