deniskuzZ commented on code in PR #6741:
URL: https://github.com/apache/hive/pull/6741#discussion_r3890287649
##########
ql/src/java/org/apache/hadoop/hive/ql/io/parquet/ParquetRecordReaderBase.java:
##########
@@ -181,6 +183,77 @@ protected ParquetInputSplit getSplit(
return split;
}
+ /**
+ * Prunes the row groups of this split with the pushed down predicate.
+ *
+ * <p>Statistics come from the footer, which is already in memory. Bloom
filters live in the data file and
+ * need an open reader, so one is opened only after statistics have had
their say and only while a
+ * surviving row group still carries a bloom filter.
+ *
+ * <p>The reader is built from a copy of the job conf with the pushed down
predicate removed. Parquet reads
+ * that predicate from the conf and filters row groups inside the
constructor; leaving it in place would
+ * read every bloom filter twice and every dictionary page once, only to
discard the result.
+ */
+ private List<BlockMetaData> filterRowGroups(FilterCompat.Filter filter,
List<BlockMetaData> splitGroup,
+ FileMetaData fileMetaData) throws IOException {
+ List<BlockMetaData> statsFiltered =
+ RowGroupFilter.filterRowGroups(filter, splitGroup,
fileMetaData.getSchema());
+ if (statsFiltered.isEmpty() ||
!jobConf.getBoolean(ParquetInputFormat.BLOOM_FILTERING_ENABLED, true) ||
+ !canBloomFilterPrune(filter) || !hasBloomFilters(statsFiltered)) {
+ return statsFiltered;
+ }
+
+ JobConf bloomFilterConf = new JobConf(jobConf);
+ bloomFilterConf.unset(ParquetInputFormat.FILTER_PREDICATE);
+ // filePath, not the split path: LLAP may rewrite it to a file id path,
and the bloom filter offsets
+ // being used here come from the footer that was read from it.
+ try (ParquetFileReader bloomFilterReader = new
ParquetFileReader(bloomFilterConf, fileMetaData, filePath,
+ statsFiltered, fileMetaData.getSchema().getColumns())) {
+ return RowGroupFilter.filterRowGroups(
+ Collections.singletonList(RowGroupFilter.FilterLevel.BLOOMFILTER),
filter, statsFiltered,
+ bloomFilterReader);
+ }
+ }
+
+ /**
+ * A bloom filter can only prove that a value is absent, so it prunes for
equality and set membership and
+ * for nothing else. Opening the data file for a predicate that cannot use
one is wasted work. Parquet
+ * reaches the same conclusion in BloomFilterImpl, but only once the reader
is already open, which is the
+ * cost this avoids.
+ */
+ private static boolean canBloomFilterPrune(FilterCompat.Filter filter) {
+ return filter instanceof FilterCompat.FilterPredicateCompat &&
+ canBloomFilterPrune(((FilterCompat.FilterPredicateCompat)
filter).getFilterPredicate());
+ }
+
+ private static boolean canBloomFilterPrune(FilterPredicate predicate) {
+ if (predicate instanceof Operators.Eq) {
+ // eq(col, null) asks for nulls, which a bloom filter says nothing about
+ return ((Operators.Eq<?>) predicate).getValue() != null;
+ }
+ if (predicate instanceof Operators.In) {
+ // Hive builds IN as a chain of ORed equalities rather than this node,
but honor it if that changes
+ return true;
+ }
+ if (predicate instanceof Operators.And) {
+ Operators.And and = (Operators.And) predicate;
+ // dropping either side of a conjunction drops the row group
+ return canBloomFilterPrune(and.getLeft()) ||
canBloomFilterPrune(and.getRight());
+ }
+ if (predicate instanceof Operators.Or) {
+ Operators.Or or = (Operators.Or) predicate;
+ // a disjunction only drops when both sides do
+ return canBloomFilterPrune(or.getLeft()) &&
canBloomFilterPrune(or.getRight());
+ }
+ return false;
+ }
+
+ private static boolean hasBloomFilters(List<BlockMetaData> blocks) {
+ return blocks.stream()
+ .flatMap(block -> block.getColumns().stream())
+ .anyMatch(column -> column.getBloomFilterOffset() > 0);
Review Comment:
it was a dummy checking whether the file had any bloom filter at all
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
