This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new b5cbf23d03 [spark] SparkFilterConverter: support AlwaysTrue/False, fix
silent NaN drop (#8060)
b5cbf23d03 is described below
commit b5cbf23d0398693d5efbc204e58aac80bb16edc8
Author: Silas <[email protected]>
AuthorDate: Thu Jun 4 12:15:13 2026 +0800
[spark] SparkFilterConverter: support AlwaysTrue/False, fix silent NaN drop
(#8060)
---
.../apache/paimon/spark/SparkFilterConverter.java | 25 ++++++++++++++--
.../paimon/spark/SparkFilterConverterTest.java | 34 ++++++++++++++++++++++
2 files changed, 56 insertions(+), 3 deletions(-)
diff --git
a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkFilterConverter.java
b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkFilterConverter.java
index c0b8cfd66b..31e4c8aec9 100644
---
a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkFilterConverter.java
+++
b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkFilterConverter.java
@@ -23,6 +23,8 @@ import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.types.DataType;
import org.apache.paimon.types.RowType;
+import org.apache.spark.sql.sources.AlwaysFalse;
+import org.apache.spark.sql.sources.AlwaysTrue;
import org.apache.spark.sql.sources.And;
import org.apache.spark.sql.sources.EqualNullSafe;
import org.apache.spark.sql.sources.EqualTo;
@@ -54,6 +56,8 @@ public class SparkFilterConverter {
public static final List<String> SUPPORT_FILTERS =
Arrays.asList(
+ "AlwaysTrue",
+ "AlwaysFalse",
"EqualTo",
"EqualNullSafe",
"GreaterThan",
@@ -97,10 +101,16 @@ public class SparkFilterConverter {
}
public Predicate convert(Filter filter) {
- if (filter instanceof EqualTo) {
+ if (filter instanceof AlwaysTrue) {
+ return PredicateBuilder.alwaysTrue();
+ } else if (filter instanceof AlwaysFalse) {
+ return PredicateBuilder.alwaysFalse();
+ } else if (filter instanceof EqualTo) {
EqualTo eq = (EqualTo) filter;
- // TODO deal with isNaN
int index = fieldIndex(eq.attribute());
+ if (isNaN(eq.value())) {
+ return builder.isNaN(index);
+ }
Object literal = convertLiteral(index, eq.value());
return builder.equal(index, literal);
} else if (filter instanceof EqualNullSafe) {
@@ -173,11 +183,20 @@ public class SparkFilterConverter {
return builder.contains(index, literal);
}
- // TODO: AlwaysTrue, AlwaysFalse
throw new UnsupportedOperationException(
filter + " is unsupported. Support Filters: " +
SUPPORT_FILTERS);
}
+ private static boolean isNaN(Object value) {
+ if (value instanceof Float) {
+ return Float.isNaN((Float) value);
+ }
+ if (value instanceof Double) {
+ return Double.isNaN((Double) value);
+ }
+ return false;
+ }
+
public Object convertLiteral(String field, Object value) {
return convertLiteral(fieldIndex(field), value);
}
diff --git
a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFilterConverterTest.java
b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFilterConverterTest.java
index 4248d07d76..8b5457c9df 100644
---
a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFilterConverterTest.java
+++
b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFilterConverterTest.java
@@ -26,11 +26,15 @@ import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.types.CharType;
import org.apache.paimon.types.DataField;
import org.apache.paimon.types.DateType;
+import org.apache.paimon.types.DoubleType;
+import org.apache.paimon.types.FloatType;
import org.apache.paimon.types.IntType;
import org.apache.paimon.types.RowType;
import org.apache.paimon.types.TimestampType;
import org.apache.paimon.types.VarCharType;
+import org.apache.spark.sql.sources.AlwaysFalse;
+import org.apache.spark.sql.sources.AlwaysTrue;
import org.apache.spark.sql.sources.EqualNullSafe;
import org.apache.spark.sql.sources.EqualTo;
import org.apache.spark.sql.sources.GreaterThan;
@@ -229,6 +233,36 @@ public class SparkFilterConverterTest {
assertThat(localDateExpression).isEqualTo(rawExpression);
}
+ @Test
+ public void testAlwaysTrueFalse() {
+ RowType rowType =
+ new RowType(Collections.singletonList(new DataField(0, "id",
new IntType())));
+ SparkFilterConverter converter = new SparkFilterConverter(rowType);
+
+ assertThat(converter.convert(new
AlwaysTrue())).isEqualTo(PredicateBuilder.alwaysTrue());
+ assertThat(converter.convert(new
AlwaysFalse())).isEqualTo(PredicateBuilder.alwaysFalse());
+ }
+
+ @Test
+ public void testEqualToNaN() {
+ RowType rowType =
+ new RowType(
+ Arrays.asList(
+ new DataField(0, "f", new FloatType()),
+ new DataField(1, "d", new DoubleType())));
+ SparkFilterConverter converter = new SparkFilterConverter(rowType);
+ PredicateBuilder builder = new PredicateBuilder(rowType);
+
+ EqualTo eqNaNFloat = EqualTo.apply("f", Float.NaN);
+ assertThat(converter.convert(eqNaNFloat)).isEqualTo(builder.isNaN(0));
+
+ EqualTo eqNaNDouble = EqualTo.apply("d", Double.NaN);
+ assertThat(converter.convert(eqNaNDouble)).isEqualTo(builder.isNaN(1));
+
+ EqualTo eqFloat = EqualTo.apply("f", 1.0f);
+ assertThat(converter.convert(eqFloat)).isEqualTo(builder.equal(0,
1.0f));
+ }
+
@Test
public void testIgnoreFailure() {
List<DataField> dataFields = new ArrayList<>();
