This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 30bfd4a600 [python] Align IN predicates with SQL null semantics (#9132)
30bfd4a600 is described below
commit 30bfd4a600a930079b15f5e6bacea34b8d3ff4c8
Author: Jingsong Lee <[email protected]>
AuthorDate: Sun Aug 9 18:49:09 2026 +0800
[python] Align IN predicates with SQL null semantics (#9132)
---
paimon-python/pypaimon/common/predicate.py | 20 +++++++++--
paimon-python/pypaimon/tests/predicates_test.py | 46 +++++++++++++++++++++++++
2 files changed, 63 insertions(+), 3 deletions(-)
diff --git a/paimon-python/pypaimon/common/predicate.py
b/paimon-python/pypaimon/common/predicate.py
index ed1f434a96..0bd1edd774 100644
--- a/paimon-python/pypaimon/common/predicate.py
+++ b/paimon-python/pypaimon/common/predicate.py
@@ -301,24 +301,38 @@ class In(Tester):
return val in literals
def test_by_stats(self, min_v, max_v, literals) -> bool:
- return any(min_v <= l <= max_v for l in literals)
+ return any(
+ min_v <= literal <= max_v
+ for literal in literals
+ if literal is not None
+ )
def test_by_arrow(self, val, literals) -> bool:
- return val.isin(literals)
+ # Arrow treats null as a set member, while SQL IN never returns true
+ # solely because both the field and an IN literal are null.
+ non_null_literals = [literal for literal in literals if literal is not
None]
+ if not non_null_literals:
+ return val.is_valid() & val.is_null()
+ return val.isin(non_null_literals) & val.is_valid()
class NotIn(Tester):
name = "notIn"
def test_by_value(self, val, literals) -> bool:
- if val is None:
+ if val is None or any(literal is None for literal in literals):
return False
return val not in literals
def test_by_stats(self, min_v, max_v, literals) -> bool:
+ if any(literal is None for literal in literals):
+ return False
return not any(min_v == l == max_v for l in literals)
def test_by_arrow(self, val, literals) -> bool:
+ # Any null literal makes SQL NOT IN unknown for every non-matching row.
+ if any(literal is None for literal in literals):
+ return val.is_valid() & val.is_null()
return (~val.isin(literals)) & val.is_valid()
diff --git a/paimon-python/pypaimon/tests/predicates_test.py
b/paimon-python/pypaimon/tests/predicates_test.py
index 1752b72f8a..ff98be7937 100644
--- a/paimon-python/pypaimon/tests/predicates_test.py
+++ b/paimon-python/pypaimon/tests/predicates_test.py
@@ -349,6 +349,18 @@ class PredicateTest(unittest.TestCase):
predicate = predicate_builder.is_in('f0', [1, 2])
_check_filtered_result(table.new_read_builder().with_filter(predicate),
self.df.loc[0:1])
+ def test_is_in_with_null_literal_append(self):
+ table = self.catalog.get_table('default.test_append')
+ predicate_builder = table.new_read_builder().new_predicate_builder()
+
+ predicate = predicate_builder.is_in('f1', [None])
+ _check_filtered_result(
+ table.new_read_builder().with_filter(predicate), self.df.iloc[0:0])
+
+ predicate = predicate_builder.is_in('f1', ['abc', None])
+ _check_filtered_result(
+ table.new_read_builder().with_filter(predicate), self.df.loc[[0]])
+
def test_is_in_pk(self):
table = self.catalog.get_table('default.test_pk')
predicate_builder = table.new_read_builder().new_predicate_builder()
@@ -361,6 +373,13 @@ class PredicateTest(unittest.TestCase):
predicate = predicate_builder.is_not_in('f0', [1, 2])
_check_filtered_result(table.new_read_builder().with_filter(predicate),
self.df.loc[2:4])
+ def test_is_not_in_with_null_literal_append(self):
+ table = self.catalog.get_table('default.test_append')
+ predicate_builder = table.new_read_builder().new_predicate_builder()
+ predicate = predicate_builder.is_not_in('f1', ['abc', None])
+ _check_filtered_result(
+ table.new_read_builder().with_filter(predicate), self.df.iloc[0:0])
+
def test_is_not_in_pk(self):
table = self.catalog.get_table('default.test_pk')
predicate_builder = table.new_read_builder().new_predicate_builder()
@@ -558,6 +577,33 @@ class PredicateTest(unittest.TestCase):
self.assertEqual(scanner.to_table().to_pydict(), {"val": [3]})
+ def test_in_and_not_in_null_literal_semantics(self):
+ table = pa.table({"val": [None, 1, 2]})
+
+ in_predicate = Predicate(
+ method='in', index=0, field='val', literals=[1, None])
+ scanner = ds.InMemoryDataset(table).scanner(
+ filter=in_predicate.to_arrow())
+ self.assertEqual(scanner.to_table().to_pydict(), {"val": [1]})
+
+ not_in_predicate = Predicate(
+ method='notIn', index=0, field='val', literals=[1, None])
+ scanner = ds.InMemoryDataset(table).scanner(
+ filter=not_in_predicate.to_arrow())
+ self.assertEqual(scanner.to_table().to_pydict(), {"val": []})
+ self.assertFalse(not_in_predicate.test(OffsetRow([2], 0, 1)))
+
+ fields = [DataField(0, 'val', 'INT')]
+ stats = SimpleStats(
+ min_values=GenericRow([1], fields),
+ max_values=GenericRow([2], fields),
+ null_counts=[0],
+ )
+ self.assertFalse(Predicate(
+ method='in', index=0, field='val', literals=[None]
+ ).test_by_simple_stats(stats, 2))
+ self.assertFalse(not_in_predicate.test_by_simple_stats(stats, 2))
+
@pytest.mark.python_plan
def test_pk_reader_with_filter(self):
pa_schema = pa.schema([
