Aggarwal-Raghav commented on code in PR #6741:
URL: https://github.com/apache/hive/pull/6741#discussion_r3890324959
##########
ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetRecordReaderBase.java:
##########
@@ -181,6 +186,95 @@ protected ParquetInputSplit getSplit(
return split;
}
+ /**
+ * Prunes the row groups of this split with the pushed down predicate.
+ *
+ * <p>Statistics come from the footer, which is already in memory. Bloom
filters live in the data file and
+ * need an open reader, so one is opened only after statistics have had
their say and only while a
+ * surviving row group still carries a bloom filter.
+ *
+ * <p>The reader is built from a copy of the job conf with the pushed down
predicate removed. Parquet reads
+ * that predicate from the conf and filters row groups inside the
constructor; leaving it in place would
+ * read every bloom filter twice and every dictionary page once, only to
discard the result.
+ */
+ private List<BlockMetaData> filterRowGroups(FilterCompat.Filter filter,
List<BlockMetaData> splitGroup,
+ FileMetaData fileMetaData) throws IOException {
+ List<BlockMetaData> statsFiltered =
+ RowGroupFilter.filterRowGroups(filter, splitGroup,
fileMetaData.getSchema());
+ Set<ColumnPath> prunableColumns = bloomFilterPrunableColumns(filter);
+ if (statsFiltered.isEmpty() ||
!jobConf.getBoolean(ParquetInputFormat.BLOOM_FILTERING_ENABLED, true) ||
+ prunableColumns.isEmpty() || !hasBloomFilters(statsFiltered,
prunableColumns)) {
+ return statsFiltered;
+ }
+
+ JobConf bloomFilterConf = new JobConf(jobConf);
+ bloomFilterConf.unset(ParquetInputFormat.FILTER_PREDICATE);
+ // filePath, not the split path: LLAP may rewrite it to a file id path,
and the bloom filter offsets
+ // being used here come from the footer that was read from it.
+ try (ParquetFileReader bloomFilterReader = new
ParquetFileReader(bloomFilterConf, fileMetaData, filePath,
+ statsFiltered, fileMetaData.getSchema().getColumns())) {
+ return RowGroupFilter.filterRowGroups(
+ Collections.singletonList(RowGroupFilter.FilterLevel.BLOOMFILTER),
filter, statsFiltered,
+ bloomFilterReader);
+ }
+ }
+
+ /**
+ * The columns this predicate could prune a row group on with a bloom
filter. A bloom filter only proves
+ * that a value is absent, so it serves equality and set membership and
nothing else, and only for the
+ * columns those terms name. An empty result means opening the data file
could not pay off. Parquet
+ * reaches the same conclusion in BloomFilterImpl, but only once the reader
is already open.
+ */
+ private static Set<ColumnPath>
bloomFilterPrunableColumns(FilterCompat.Filter filter) {
+ return filter instanceof FilterCompat.FilterPredicateCompat ?
+ bloomFilterPrunableColumns(((FilterCompat.FilterPredicateCompat)
filter).getFilterPredicate()) :
+ Collections.emptySet();
+ }
+
+ private static Set<ColumnPath> bloomFilterPrunableColumns(FilterPredicate
predicate) {
+ if (predicate instanceof Operators.Eq) {
+ Operators.Eq<?> eq = (Operators.Eq<?>) predicate;
+ // eq(col, null) asks for nulls, which a bloom filter says nothing about
+ return eq.getValue() == null ? Collections.emptySet() :
+ Collections.singleton(eq.getColumn().getColumnPath());
+ }
+ if (predicate instanceof Operators.In) {
+ // Hive builds IN as a chain of ORed equalities rather than this node,
but honor it if that changes
+ return Collections.singleton(((Operators.In<?>)
predicate).getColumn().getColumnPath());
+ }
+ if (predicate instanceof Operators.And) {
+ Operators.And and = (Operators.And) predicate;
+ // dropping either side of a conjunction drops the row group
+ return union(bloomFilterPrunableColumns(and.getLeft()),
bloomFilterPrunableColumns(and.getRight()));
+ }
+ if (predicate instanceof Operators.Or) {
Review Comment:
Maybe use switch expression as well?
```
private static Set<ColumnPath> bloomFilterPrunableColumns(FilterPredicate
predicate) {
return switch (predicate) {
case Operators.Eq<?> eq ->
// eq(col, null) asks for nulls, which a bloom filter says nothing
about
eq.getValue() == null ? Collections.emptySet() :
Collections.singleton(eq.getColumn().getColumnPath());
case Operators.In<?> in ->
// Hive builds IN as a chain of ORed equalities rather than this
node, but honor it if that changes
Collections.singleton(in.getColumn().getColumnPath());
case Operators.And and ->
// dropping either side of a conjunction drops the row group
union(bloomFilterPrunableColumns(and.getLeft()),
bloomFilterPrunableColumns(and.getRight()));
case Operators.Or or -> {
Set<ColumnPath> left = bloomFilterPrunableColumns(or.getLeft());
Set<ColumnPath> right = bloomFilterPrunableColumns(or.getRight());
// a disjunction only drops when both sides do
yield left.isEmpty() || right.isEmpty() ? Collections.emptySet() :
union(left, right);
}
default -> Collections.emptySet();
};
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
