Repository: hive Updated Branches: refs/heads/master 9a511eb97 -> 8ef6e6835
HIVE-11033: BloomFilter index is not honored by ORC reader (Prasanth Jayachandran reviewed by Gopal V) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8ef6e683 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8ef6e683 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8ef6e683 Branch: refs/heads/master Commit: 8ef6e6835356989c9a36d2dfdbcfd71c675ced9b Parents: 9a511eb Author: Prasanth Jayachandran <[email protected]> Authored: Thu Jun 18 01:08:38 2015 -0700 Committer: Prasanth Jayachandran <[email protected]> Committed: Thu Jun 18 01:08:38 2015 -0700 ---------------------------------------------------------------------- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java | 4 ++++ .../org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java | 9 ++++----- 2 files changed, 8 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/8ef6e683/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java index 0776018..db2ca15 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java @@ -57,6 +57,10 @@ public class OrcUtils { ObjectInspector inspector) { int numFlattenedCols = getFlattenedColumnsCount(inspector); boolean[] results = new boolean[numFlattenedCols]; + if ("*".equals(selectedColumns)) { + Arrays.fill(results, true); + return results; + } if (selectedColumns != null && !selectedColumns.isEmpty()) { includeColumnsImpl(results, selectedColumns.toLowerCase(), allColumns, inspector); } http://git-wip-us.apache.org/repos/asf/hive/blob/8ef6e683/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java index 58e19cb..beaf231 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java @@ -666,7 +666,6 @@ class RecordReaderImpl implements RecordReader { private final List<PredicateLeaf> sargLeaves; private final int[] filterColumns; private final long rowIndexStride; - private final OrcProto.BloomFilterIndex[] bloomFilterIndices; // same as the above array, but indices are set to true private final boolean[] sargColumns; public SargApplier(SearchArgument sarg, String[] columnNames, long rowIndexStride, @@ -674,7 +673,6 @@ class RecordReaderImpl implements RecordReader { this.sarg = sarg; sargLeaves = sarg.getLeaves(); filterColumns = mapSargColumns(sargLeaves, columnNames, 0); - bloomFilterIndices = new OrcProto.BloomFilterIndex[types.size()]; this.rowIndexStride = rowIndexStride; // included will not be null, row options will fill the array with trues if null sargColumns = new boolean[includedCount]; @@ -694,7 +692,8 @@ class RecordReaderImpl implements RecordReader { * @throws IOException */ public boolean[] pickRowGroups( - StripeInformation stripe, OrcProto.RowIndex[] indexes) throws IOException { + StripeInformation stripe, OrcProto.RowIndex[] indexes, + OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException { long rowsInStripe = stripe.getNumberOfRows(); int groupsInStripe = (int) ((rowsInStripe + rowIndexStride - 1) / rowIndexStride); boolean[] result = new boolean[groupsInStripe]; // TODO: avoid alloc? @@ -705,7 +704,7 @@ class RecordReaderImpl implements RecordReader { OrcProto.ColumnStatistics stats = indexes[filterColumns[pred]].getEntry(rowGroup).getStatistics(); OrcProto.BloomFilter bf = null; - if (bloomFilterIndices[filterColumns[pred]] != null) { + if (bloomFilterIndices != null && bloomFilterIndices[filterColumns[pred]] != null) { bf = bloomFilterIndices[filterColumns[pred]].getBloomFilter(rowGroup); } leafValues[pred] = evaluatePredicateProto(stats, sargLeaves.get(pred), bf); @@ -749,7 +748,7 @@ class RecordReaderImpl implements RecordReader { return null; } readRowIndex(currentStripe, included, sargApp.sargColumns); - return sargApp.pickRowGroups(stripes.get(currentStripe), indexes); + return sargApp.pickRowGroups(stripes.get(currentStripe), indexes, bloomFilterIndices); } private void clearStreams() throws IOException {
