This is an automated email from the ASF dual-hosted git repository.
abstractdog pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 0745aeadd66 HIVE-30182: Parquet: a leaf without its runtime value
drops the whole row-group filter (#6954)
0745aeadd66 is described below
commit 0745aeadd6686739d85f360456614618affc7c1c
Author: Bodor Laszlo <[email protected]>
AuthorDate: Fri Nov 20 12:27:27 2026 +0100
HIVE-30182: Parquet: a leaf without its runtime value drops the whole
row-group filter (#6954)
---
.../read/ParquetFilterPredicateConverter.java | 13 ++-
.../parquet/read/TestParquetFilterPredicate.java | 107 +++++++++++++++++++++
2 files changed, 119 insertions(+), 1 deletion(-)
diff --git
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetFilterPredicateConverter.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetFilterPredicateConverter.java
index 68d5fbf1cb1..3fa718b76b5 100644
---
a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetFilterPredicateConverter.java
+++
b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetFilterPredicateConverter.java
@@ -20,6 +20,7 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.hive.common.NoDynamicValuesException;
import org.apache.hadoop.hive.ql.io.parquet.FilterPredicateLeafBuilder;
import org.apache.hadoop.hive.ql.io.parquet.LeafFilterFactory;
import org.apache.hadoop.hive.ql.io.sarg.ExpressionTree;
@@ -131,8 +132,15 @@ private static FilterPredicate translate(ExpressionTree
root,
.getLeafFilterBuilderByType(leaf.getType(), parquetType);
if (isMultiLiteralsOperator(leaf.getOperator())) {
+ List<Object> literals = leaf.getLiteralList();
+ // getLiteralList() drops null runtime values, and returns an empty
list when they are not available;
+ // a BETWEEN without both bounds cannot filter either
+ if (literals.isEmpty() || (leaf.getOperator() ==
PredicateLeaf.Operator.BETWEEN && literals.size() != 2)) {
+ LOG.debug("Leaf {} has no usable literals {}, it filters nothing",
leaf, literals);
+ return null;
+ }
return builder.buildPredicate(leaf.getOperator(),
- leaf.getLiteralList(),
+ literals,
leaf.getColumnName(),
columnType);
} else {
@@ -142,6 +150,9 @@ private static FilterPredicate translate(ExpressionTree
root,
leaf.getColumnName(),
columnType);
}
+ } catch (NoDynamicValuesException e) {
+ LOG.debug("Runtime value of leaf {} is not available, it filters
nothing", leaf, e);
+ return null;
} catch (Exception e) {
LOG.error("fail to build predicate filter leaf with errors" + e, e);
throw e;
diff --git
a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/read/TestParquetFilterPredicate.java
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/read/TestParquetFilterPredicate.java
index 881f093b8ff..c8ddac75281 100644
---
a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/read/TestParquetFilterPredicate.java
+++
b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/read/TestParquetFilterPredicate.java
@@ -20,10 +20,14 @@
package org.apache.hadoop.hive.ql.io.parquet.read;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.common.NoDynamicValuesException;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.ql.io.parquet.LeafFilterFactory;
+import org.apache.hadoop.hive.ql.io.sarg.LiteralDelegate;
import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
@@ -43,6 +47,7 @@
import java.sql.Timestamp;
import java.util.HashMap;
import java.util.Map;
+import java.util.function.Supplier;
public class TestParquetFilterPredicate {
@@ -736,4 +741,106 @@ public void testFilterMoreComplexVarCharColumn() throws
Exception {
+ "or(eq(c, Binary{\"banana\"}), lt(d, Binary{\"cherry\"})))";
assertEquals(expected, p.toString());
}
+
+ /** A single-literal leaf whose runtime value is not available is dropped;
the rest of the conjunction filters. */
+ @Test
+ public void testUnresolvedSingleLiteralDropsOnlyItsLeaf() {
+ SearchArgument sarg = SearchArgumentFactory.newBuilder()
+ .startAnd()
+ .lessThan("a", PredicateLeaf.Type.LONG, 5L)
+ .equals("b", PredicateLeaf.Type.LONG, unresolved())
+ .end()
+ .build();
+ assertEquals("lt(a, 5)", String.valueOf(toFilterPredicateOverAbc(sarg)));
+ }
+
+ /** A BETWEEN leaf whose runtime values are not available is dropped; the
rest of the conjunction filters. */
+ @Test
+ public void testUnresolvedBetweenDropsOnlyItsLeaf() {
+ SearchArgument sarg = SearchArgumentFactory.newBuilder()
+ .startAnd()
+ .lessThan("a", PredicateLeaf.Type.LONG, 5L)
+ .between("c", PredicateLeaf.Type.LONG, unresolved(), unresolved())
+ .end()
+ .build();
+ assertEquals("lt(a, 5)", String.valueOf(toFilterPredicateOverAbc(sarg)));
+ }
+
+ /** A BETWEEN leaf with only one of its runtime bounds resolved to a value
is dropped. */
+ @Test
+ public void testBetweenWithOneNullRuntimeBoundDropsOnlyItsLeaf() {
+ SearchArgument sarg = SearchArgumentFactory.newBuilder()
+ .startAnd()
+ .lessThan("a", PredicateLeaf.Type.LONG, 5L)
+ .between("c", PredicateLeaf.Type.LONG, resolvedTo(null),
resolvedTo(10L))
+ .end()
+ .build();
+ assertEquals("lt(a, 5)", String.valueOf(toFilterPredicateOverAbc(sarg)));
+ }
+
+ /** A disjunction containing an unresolved leaf filters nothing. */
+ @Test
+ public void testUnresolvedLeafInDisjunctionFiltersNothing() {
+ SearchArgument sarg = SearchArgumentFactory.newBuilder()
+ .startOr()
+ .lessThan("a", PredicateLeaf.Type.LONG, 5L)
+ .between("c", PredicateLeaf.Type.LONG, unresolved(), unresolved())
+ .end()
+ .build();
+ assertNull(toFilterPredicateOverAbc(sarg));
+ }
+
+ /** A runtime value (a DynamicValue) that is not available when the reader
builds its filter. */
+ private static LiteralDelegate unresolved() {
+ return new RuntimeValue("unresolved", () -> {
+ throw new NoDynamicValuesException("not available");
+ });
+ }
+
+ /** A runtime value (a DynamicValue) that resolved to the given value. */
+ private static LiteralDelegate resolvedTo(Object value) {
+ return new RuntimeValue("resolved", () -> value);
+ }
+
+ /** A runtime value that keeps its configuration, like a DynamicValue. */
+ private static final class RuntimeValue implements LiteralDelegate {
+ private final String id;
+ private final Supplier<Object> value;
+ private Configuration conf;
+
+ RuntimeValue(String id, Supplier<Object> value) {
+ this.id = id;
+ this.value = value;
+ }
+
+ @Override
+ public Object getLiteral() {
+ return value.get();
+ }
+
+ @Override
+ public String getId() {
+ return id;
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+ }
+
+ private static FilterPredicate toFilterPredicateOverAbc(SearchArgument sarg)
{
+ MessageType schema = MessageTypeParser.parseMessageType(
+ "message test { required int32 a; required int32 b; required int32 c;
}");
+ Map<String, TypeInfo> columnTypes = new HashMap<>();
+ for (String column : new String[] {"a", "b", "c"}) {
+ columnTypes.put(column, TypeInfoFactory.getPrimitiveTypeInfo("int"));
+ }
+ return ParquetFilterPredicateConverter.toFilterPredicate(sarg, schema,
columnTypes);
+ }
}