This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 8336f1aecb [format] Decline ORC pushdown of empty IN/NOT IN instead of
crashing the reader (#10015)
8336f1aecb is described below
commit 8336f1aecb9873075bf0091487122da8e0518132
Author: Xiangyi Zhu <[email protected]>
AuthorDate: Mon Sep 21 10:40:51 2026 +0800
[format] Decline ORC pushdown of empty IN/NOT IN instead of crashing the
reader (#10015)
---
.../orc/filter/OrcPredicateFunctionVisitor.java | 7 ++++
.../paimon/format/orc/OrcFormatReadWriteTest.java | 47 ++++++++++++++++++++++
.../format/orc/filter/OrcFilterConverterTest.java | 43 ++++++++++++++++++++
3 files changed, 97 insertions(+)
diff --git
a/paimon-format/src/main/java/org/apache/paimon/format/orc/filter/OrcPredicateFunctionVisitor.java
b/paimon-format/src/main/java/org/apache/paimon/format/orc/filter/OrcPredicateFunctionVisitor.java
index 3ee59e2c50..1e5825f96a 100644
---
a/paimon-format/src/main/java/org/apache/paimon/format/orc/filter/OrcPredicateFunctionVisitor.java
+++
b/paimon-format/src/main/java/org/apache/paimon/format/orc/filter/OrcPredicateFunctionVisitor.java
@@ -174,6 +174,13 @@ public class OrcPredicateFunctionVisitor
@Override
public Optional<OrcFilters.Predicate> visitIn(FieldRef fieldRef,
List<Object> literals) {
+ if (literals.isEmpty()) {
+ // An IN predicate builder can legitimately produce an empty
(always-false) leaf, but
+ // Hive's SearchArgument.Builder.in(...) rejects an empty set
outright. Decline the
+ // pushdown; residual evaluation upstream still applies the
correct semantics.
+ return Optional.empty();
+ }
+
PredicateLeaf.Type colType = toOrcType(fieldRef.type());
if (colType == null) {
return Optional.empty();
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java
index c18c462a99..d80debf9be 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFormatReadWriteTest.java
@@ -45,6 +45,7 @@ import java.nio.charset.StandardCharsets;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -202,6 +203,52 @@ public class OrcFormatReadWriteTest extends
FormatReadWriteTest {
assertThat(result).containsExactly(GenericRow.of(localTimestamp));
}
+ /**
+ * {@link org.apache.paimon.predicate.PredicateBuilder#in(int, List)} on
an empty literal list
+ * is a legitimate always-false leaf, but Hive's {@code
SearchArgument.Builder.in(...)} rejects
+ * a zero-length call outright ({@code IllegalArgumentException("Can't
create in expression with
+ * no arguments")}). Opening a reader with such a predicate must not
crash; the always-false (or
+ * always-true, for NOT IN) semantics are enforced by residual evaluation
upstream, not by
+ * ORC-level pruning.
+ */
+ @Test
+ public void testEmptyInAndNotInPredicatesDoNotCrashTheReader() throws
IOException {
+ RowType rowType = DataTypes.ROW(DataTypes.FIELD(0, "id",
DataTypes.BIGINT()));
+ write(
+ fileFormat().createWriterFactory(rowType),
+ file,
+ GenericRow.of(1L),
+ GenericRow.of(2L));
+
+ org.apache.paimon.predicate.Predicate emptyIn =
+ new org.apache.paimon.predicate.PredicateBuilder(rowType)
+ .in(0, Collections.emptyList());
+ org.apache.paimon.predicate.Predicate emptyNotIn =
+ new org.apache.paimon.predicate.PredicateBuilder(rowType)
+ .notIn(0, Collections.emptyList());
+
+ for (org.apache.paimon.predicate.Predicate predicate :
Arrays.asList(emptyIn, emptyNotIn)) {
+ List<org.apache.paimon.predicate.Predicate> filters = new
ArrayList<>();
+ filters.add(predicate);
+ try (RecordReader<InternalRow> reader =
+ fileFormat()
+ .createReaderFactory(rowType, rowType, filters)
+ .createReader(
+ new FormatReaderContext(
+ fileIO, file,
fileIO.getFileSize(file), null, null))) {
+ int count = 0;
+ RecordReader.RecordIterator<InternalRow> batch;
+ while ((batch = reader.readBatch()) != null) {
+ while (batch.next() != null) {
+ count++;
+ }
+ batch.releaseBatch();
+ }
+ assertThat(count).as(predicate.toString()).isEqualTo(2);
+ }
+ }
+ }
+
@Test
public void testTimestampLTZWithNewWriteAndLegacyRead() throws IOException
{
RowType rowType =
DataTypes.ROW(DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE());
diff --git
a/paimon-format/src/test/java/org/apache/paimon/format/orc/filter/OrcFilterConverterTest.java
b/paimon-format/src/test/java/org/apache/paimon/format/orc/filter/OrcFilterConverterTest.java
index 03b3e88727..ead0762b2a 100644
---
a/paimon-format/src/test/java/org/apache/paimon/format/orc/filter/OrcFilterConverterTest.java
+++
b/paimon-format/src/test/java/org/apache/paimon/format/orc/filter/OrcFilterConverterTest.java
@@ -237,6 +237,49 @@ public class OrcFilterConverterTest {
true);
}
+ /**
+ * {@link PredicateBuilder#in(int, List)} special-cases an empty literal
list the same way it
+ * special-cases more than 20, building a raw {@code In} leaf rather than
an OR chain - so an
+ * empty list reaches {@link OrcPredicateFunctionVisitor#visitIn} exactly
like the many-values
+ * case above. Unlike parquet-mr, Hive's own {@code
SearchArgument.Builder.in(...)} rejects a
+ * zero-length varargs call with {@code IllegalArgumentException("Can't
create in expression
+ * with no arguments")}, so this predicate must never reach it - the
visitor has to decline the
+ * pushdown for an empty literal list instead of building an {@code
OrcFilters.In} with an empty
+ * array.
+ */
+ @Test
+ public void testInPredicateWithEmptyValuesIsNotPushedDown() {
+ PredicateBuilder builder =
+ new PredicateBuilder(
+ new RowType(
+ Collections.singletonList(
+ new DataField(0, "testField", new
BigIntType()))));
+
+ assertThat(
+ builder.in(0, Collections.emptyList())
+ .visit(OrcPredicateFunctionVisitor.VISITOR))
+ .isEqualTo(Optional.empty());
+ assertThat(
+ builder.notIn(0, Collections.emptyList())
+ .visit(OrcPredicateFunctionVisitor.VISITOR))
+ .isEqualTo(Optional.empty());
+
+ // or()/and() only fold away AlwaysFalse.INSTANCE, not an empty In
leaf, so the leaf
+ // survives into a compound; a declined child must decline the whole
compound, not crash.
+ assertThat(
+ PredicateBuilder.or(
+ builder.in(0, Collections.emptyList()),
+ builder.equal(0, 1L))
+ .visit(OrcPredicateFunctionVisitor.VISITOR))
+ .isEqualTo(Optional.empty());
+ assertThat(
+ PredicateBuilder.and(
+ builder.notIn(0,
Collections.emptyList()),
+ builder.equal(0, 1L))
+ .visit(OrcPredicateFunctionVisitor.VISITOR))
+ .isEqualTo(Optional.empty());
+ }
+
@Test
public void testIsNaN() {
PredicateBuilder builder =
