This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 5d2370bf7e [python] Match no rows for a null literal in comparison
predicates (#10814)
5d2370bf7e is described below
commit 5d2370bf7e5baddf20ea47df34001110cd45178d
Author: Jiajia Li <[email protected]>
AuthorDate: Tue Oct 27 14:14:02 2026 +0800
[python] Match no rows for a null literal in comparison predicates (#10814)
---
paimon-python/pypaimon/common/predicate.py | 74 +++++++++++++++++-----
.../pypaimon/table/system/system_table_read.py | 13 ----
paimon-python/pypaimon/tests/predicates_test.py | 65 +++++++++++++++++++
.../tests/system/system_string_filter_test.py | 4 +-
4 files changed, 126 insertions(+), 30 deletions(-)
diff --git a/paimon-python/pypaimon/common/predicate.py
b/paimon-python/pypaimon/common/predicate.py
index c6e14ae573..6bd6801e6f 100644
--- a/paimon-python/pypaimon/common/predicate.py
+++ b/paimon-python/pypaimon/common/predicate.py
@@ -169,6 +169,12 @@ class Predicate:
lambda left, right: left | right,
)
+ tester = Predicate.testers.get(self.method)
+ if tester is not None and tester.literal_arity and
_null_literal(self.literals, tester.literal_arity):
+ # Arrow would keep `x < a OR x > NULL` for x < a, but a null
literal matches no row
+ field = pyarrow_dataset.field(self.field)
+ return field.is_valid() & field.is_null()
+
if self.method == 'startsWith':
pattern = self.literals[0]
# For PyArrow compatibility - improved approach
@@ -275,6 +281,8 @@ class RegisterMeta(ABCMeta):
class Tester(ABC, metaclass=RegisterMeta):
name = None
+ # number of literals compared like Java LeafBinaryFunction (1) or
LeafTernaryFunction (2)
+ literal_arity = 0
@abstractmethod
def test_by_value(self, val, literals) -> bool:
@@ -295,15 +303,24 @@ class Tester(ABC, metaclass=RegisterMeta):
"""
+def _null_literal(literals, arity=1) -> bool:
+ """A null literal matches no row, as in Java LeafBinaryFunction and
LeafTernaryFunction."""
+ return not literals or len(literals) < arity or any(
+ literal is None for literal in literals[:arity])
+
+
class Equal(Tester):
name = 'equal'
+ literal_arity = 1
def test_by_value(self, val, literals) -> bool:
- if val is None or not literals:
+ if val is None or _null_literal(literals):
return False
return val == literals[0]
def test_by_stats(self, min_v, max_v, literals) -> bool:
+ if _null_literal(literals):
+ return False
return min_v <= literals[0] <= max_v
def test_by_arrow(self, val, literals) -> bool:
@@ -312,13 +329,16 @@ class Equal(Tester):
class NotEqual(Tester):
name = "notEqual"
+ literal_arity = 1
def test_by_value(self, val, literals) -> bool:
- if val is None or not literals:
+ if val is None or _null_literal(literals):
return False
return val != literals[0]
def test_by_stats(self, min_v, max_v, literals) -> bool:
+ if _null_literal(literals):
+ return False
return not (min_v == literals[0] == max_v)
def test_by_arrow(self, val, literals) -> bool:
@@ -327,13 +347,16 @@ class NotEqual(Tester):
class LessThan(Tester):
name = "lessThan"
+ literal_arity = 1
def test_by_value(self, val, literals) -> bool:
- if val is None or not literals:
+ if val is None or _null_literal(literals):
return False
return val < literals[0]
def test_by_stats(self, min_v, max_v, literals) -> bool:
+ if _null_literal(literals):
+ return False
return literals[0] > min_v
def test_by_arrow(self, val, literals) -> bool:
@@ -342,13 +365,16 @@ class LessThan(Tester):
class LessOrEqual(Tester):
name = "lessOrEqual"
+ literal_arity = 1
def test_by_value(self, val, literals) -> bool:
- if val is None or not literals:
+ if val is None or _null_literal(literals):
return False
return val <= literals[0]
def test_by_stats(self, min_v, max_v, literals) -> bool:
+ if _null_literal(literals):
+ return False
return literals[0] >= min_v
def test_by_arrow(self, val, literals) -> bool:
@@ -357,13 +383,16 @@ class LessOrEqual(Tester):
class GreaterThan(Tester):
name = "greaterThan"
+ literal_arity = 1
def test_by_value(self, val, literals) -> bool:
- if val is None or not literals:
+ if val is None or _null_literal(literals):
return False
return val > literals[0]
def test_by_stats(self, min_v, max_v, literals) -> bool:
+ if _null_literal(literals):
+ return False
return literals[0] < max_v
def test_by_arrow(self, val, literals) -> bool:
@@ -372,13 +401,16 @@ class GreaterThan(Tester):
class GreaterOrEqual(Tester):
name = "greaterOrEqual"
+ literal_arity = 1
def test_by_value(self, val, literals) -> bool:
- if val is None or not literals:
+ if val is None or _null_literal(literals):
return False
return val >= literals[0]
def test_by_stats(self, min_v, max_v, literals) -> bool:
+ if _null_literal(literals):
+ return False
return literals[0] <= max_v
def test_by_arrow(self, val, literals) -> bool:
@@ -431,13 +463,16 @@ class NotIn(Tester):
class Between(Tester):
name = "between"
+ literal_arity = 2
def test_by_value(self, val, literals) -> bool:
- if val is None or not literals or len(literals) < 2:
+ if val is None or _null_literal(literals, 2):
return False
return literals[0] <= val <= literals[1]
def test_by_stats(self, min_v, max_v, literals) -> bool:
+ if _null_literal(literals, 2):
+ return False
return literals[0] <= max_v and literals[1] >= min_v
def test_by_arrow(self, val, literals) -> bool:
@@ -446,13 +481,16 @@ class Between(Tester):
class StartsWith(Tester):
name = "startsWith"
+ literal_arity = 1
def test_by_value(self, val, literals) -> bool:
- if val is None or not literals:
+ if val is None or _null_literal(literals):
return False
return isinstance(val, str) and val.startswith(literals[0])
def test_by_stats(self, min_v, max_v, literals) -> bool:
+ if _null_literal(literals):
+ return False
return ((isinstance(min_v, str) and isinstance(max_v, str)) and
((min_v.startswith(literals[0]) or min_v < literals[0]) and
(max_v.startswith(literals[0]) or max_v > literals[0])))
@@ -463,14 +501,15 @@ class StartsWith(Tester):
class EndsWith(Tester):
name = "endsWith"
+ literal_arity = 1
def test_by_value(self, val, literals) -> bool:
- if val is None or not literals:
+ if val is None or _null_literal(literals):
return False
return isinstance(val, str) and val.endswith(literals[0])
def test_by_stats(self, min_v, max_v, literals) -> bool:
- return True
+ return not _null_literal(literals)
def test_by_arrow(self, val, literals) -> bool:
return True
@@ -478,14 +517,15 @@ class EndsWith(Tester):
class Contains(Tester):
name = "contains"
+ literal_arity = 1
def test_by_value(self, val, literals) -> bool:
- if val is None or not literals:
+ if val is None or _null_literal(literals):
return False
return isinstance(val, str) and literals[0] in val
def test_by_stats(self, min_v, max_v, literals) -> bool:
- return True
+ return not _null_literal(literals)
def test_by_arrow(self, val, literals) -> bool:
return True
@@ -519,13 +559,16 @@ class IsNotNull(Tester):
class NotBetween(Tester):
name = "notBetween"
+ literal_arity = 2
def test_by_value(self, val, literals) -> bool:
- if val is None or not literals or len(literals) < 2:
+ if val is None or _null_literal(literals, 2):
return False
return val < literals[0] or val > literals[1]
def test_by_stats(self, min_v, max_v, literals) -> bool:
+ if _null_literal(literals, 2):
+ return False
return literals[0] > min_v or literals[1] < max_v
def test_by_arrow(self, val, literals) -> bool:
@@ -534,6 +577,7 @@ class NotBetween(Tester):
class Like(Tester):
name = "like"
+ literal_arity = 1
@staticmethod
def _sql_like_to_regex(pattern: str, escape_char: str = '\\') -> str:
@@ -566,7 +610,7 @@ class Like(Tester):
return ''.join(regex_parts)
def test_by_value(self, val, literals) -> bool:
- if val is None or not literals:
+ if val is None or _null_literal(literals):
return False
if not isinstance(val, str):
return False
@@ -574,7 +618,7 @@ class Like(Tester):
return bool(re.fullmatch(pattern, val))
def test_by_stats(self, min_v, max_v, literals) -> bool:
- return True
+ return not _null_literal(literals)
def test_by_arrow(self, val, literals) -> bool:
return True
diff --git a/paimon-python/pypaimon/table/system/system_table_read.py
b/paimon-python/pypaimon/table/system/system_table_read.py
index ece7312d6b..61944f7991 100644
--- a/paimon-python/pypaimon/table/system/system_table_read.py
+++ b/paimon-python/pypaimon/table/system/system_table_read.py
@@ -169,19 +169,6 @@ class SystemTableRead:
if predicate.method in ("and", "or"):
results = (self._test_row(child, row) for child in
predicate.literals)
return all(results) if predicate.method == "and" else any(results)
- if predicate.method in (
- "equal", "notEqual", "lessThan", "lessOrEqual", "greaterThan",
"greaterOrEqual"
- ) and predicate.literals[0] is None:
- # Arrow comparisons with NULL never select a row.
- return False
- if predicate.method in ("between", "notBetween") and any(v is None for
v in predicate.literals):
- value = row.get_field(predicate.index)
- if predicate.method == "between" or value is None:
- return False
- # NOT BETWEEN is an OR: a non-null bound can still match.
- lower, upper = predicate.literals
- return ((lower is not None and value < lower)
- or (upper is not None and value > upper))
if predicate.method == "like":
value = row.get_field(predicate.index)
# SQL '_' matches a newline too, as in Java's Like predicate.
diff --git a/paimon-python/pypaimon/tests/predicates_test.py
b/paimon-python/pypaimon/tests/predicates_test.py
index 918c95c578..3e07631c76 100644
--- a/paimon-python/pypaimon/tests/predicates_test.py
+++ b/paimon-python/pypaimon/tests/predicates_test.py
@@ -442,6 +442,71 @@ class PredicateTest(unittest.TestCase):
predicate = predicate_builder.like('f1', '%bc')
_check_filtered_result(table.new_read_builder().with_filter(predicate),
self.df.loc[0:2])
+ def test_null_literal_matches_nothing(self):
+ pa_schema = pa.schema([('f0', pa.int64()), ('f1', pa.string())])
+ # without stats no file is pruned, so the Arrow filter decides
+ self.catalog.create_table('default.test_append_no_stats',
Schema.from_pyarrow_schema(
+ pa_schema, options={'metadata.stats-mode': 'none'}), False)
+ # two commits to one bucket, so the read merges and filters row by row
+ self.catalog.create_table('default.test_pk_merge',
Schema.from_pyarrow_schema(
+ pa_schema, primary_keys=['f0'], options={'bucket': '1'}), False)
+ for table_name, dfs in (('default.test_append_no_stats', [self.df]),
+ ('default.test_pk_merge', [self.df,
self.df.loc[[0]]])):
+ table = self.catalog.get_table(table_name)
+ for df in dfs:
+ write_builder = table.new_batch_write_builder()
+ write = write_builder.new_write()
+ commit = write_builder.new_commit()
+ write.write_pandas(df)
+ commit.commit(write.prepare_commit())
+ write.close()
+ commit.close()
+
+ for table_name in ('default.test_append',
'default.test_append_no_stats',
+ 'default.test_pk', 'default.test_pk_merge'):
+ table = self.catalog.get_table(table_name)
+ predicate_builder =
table.new_read_builder().new_predicate_builder()
+ predicates = [
+ predicate_builder.equal('f1', None),
+ predicate_builder.not_equal('f1', None),
+ predicate_builder.less_than('f1', None),
+ predicate_builder.less_or_equal('f1', None),
+ predicate_builder.greater_than('f1', None),
+ predicate_builder.greater_or_equal('f1', None),
+ predicate_builder.between('f1', None, 'b'),
+ predicate_builder.not_between('f1', 'b', None),
+ predicate_builder.startswith('f1', None),
+ predicate_builder.endswith('f1', None),
+ predicate_builder.contains('f1', None),
+ predicate_builder.like('f1', None),
+ ]
+ for predicate in predicates:
+ with self.subTest(table=table_name, method=predicate.method):
+ _check_filtered_result(
+ table.new_read_builder().with_filter(predicate),
self.df.iloc[0:0])
+
+ def test_null_literal_by_value_stats_and_arrow(self):
+ fields = [DataField(0, 'f0', 'STRING')]
+ record = GenericRow(['None'], fields)
+ stats = SimpleStats(
+ min_values=GenericRow(['a'], fields),
+ max_values=GenericRow(['z'], fields),
+ null_counts=[0],
+ )
+ batch = pa.table({'f0': ['None', 'a', 'z', None]})
+ for method, literals in (('equal', [None]), ('notEqual', [None]),
('lessThan', [None]),
+ ('lessOrEqual', [None]), ('greaterThan',
[None]),
+ ('greaterOrEqual', [None]), ('between',
[None, 'z']),
+ ('notBetween', ['a', None]), ('startsWith',
[None]),
+ ('endsWith', [None]), ('contains', [None]),
('like', [None])):
+ predicate = Predicate(method=method, index=0, field='f0',
literals=literals)
+ with self.subTest(method=method):
+ # like must not read a null pattern as the string 'None'
+ self.assertFalse(predicate.test(record))
+ self.assertFalse(predicate.test_by_simple_stats(stats, 10))
+ self.assertEqual(0, ds.dataset(batch).to_table(
+ filter=predicate.to_arrow(batch.schema)).num_rows)
+
def test_and_predicates(self):
table = self.catalog.get_table('default.test_append')
predicate_builder = table.new_read_builder().new_predicate_builder()
diff --git a/paimon-python/pypaimon/tests/system/system_string_filter_test.py
b/paimon-python/pypaimon/tests/system/system_string_filter_test.py
index bc200fb407..191fd7aea2 100644
--- a/paimon-python/pypaimon/tests/system/system_string_filter_test.py
+++ b/paimon-python/pypaimon/tests/system/system_string_filter_test.py
@@ -92,8 +92,8 @@ def
test_mixed_null_comparisons_preserve_arrow_semantics(table, method, field, c
("between", [None, 3], []),
("between", [3, None], []),
("between", [None, None], []),
- ("not_between", [None, 3], [4, 5, 6, 7, 8, 9, 10]),
- ("not_between", [3, None], [1, 2]),
+ ("not_between", [None, 3], []),
+ ("not_between", [3, None], []),
("not_between", [None, None], []),
("is_in", [[None, 2]], [2]),
("is_in", [[None]], []),
