This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/paimon-rust.git
The following commit(s) were added to refs/heads/main by this push:
new c31381b [python] Push down like/startsWith/endsWith/contains
predicates (#445)
c31381b is described below
commit c31381b85c03d59d0eb834d696af98f40bc3abe5
Author: Junrui Lee <[email protected]>
AuthorDate: Fri Jul 3 18:47:20 2026 +0800
[python] Push down like/startsWith/endsWith/contains predicates (#445)
---
bindings/python/src/predicate.rs | 221 ++++++++++++++++++++++++++++++++++---
bindings/python/tests/test_read.py | 98 ++++++++++++++--
2 files changed, 294 insertions(+), 25 deletions(-)
diff --git a/bindings/python/src/predicate.rs b/bindings/python/src/predicate.rs
index cb2c852..0e9962b 100644
--- a/bindings/python/src/predicate.rs
+++ b/bindings/python/src/predicate.rs
@@ -105,7 +105,7 @@ fn float_val(value: &Bound<'_, PyAny>) -> PyResult<f64> {
/// Operators recognized by the lightweight dict format but not translatable
to a
/// Rust [`Predicate`] for pushdown.
-const METHOD_NOT_SUPPORTED: &[&str] = &["like", "startsWith", "endsWith",
"contains", "not"];
+const METHOD_NOT_SUPPORTED: &[&str] = &["not"];
/// Recursively convert a lightweight dict predicate into a Rust [`Predicate`].
///
@@ -242,6 +242,25 @@ fn leaf_to_predicate(
}
pb.is_not_in(&field, ds)
}
+ "startsWith" => pb.starts_with(&field, one(to_datums(literals_obj)?)?),
+ "endsWith" => pb.ends_with(&field, one(to_datums(literals_obj)?)?),
+ "contains" => pb.contains(&field, one(to_datums(literals_obj)?)?),
+ "like" => {
+ // 1 literal: pattern with the default '\' escape.
+ // 2 literals: [pattern, escape] where escape is a single character
+ // (SQL `LIKE .. ESCAPE ..`).
+ let mut ds = to_datums(literals_obj)?;
+ let escape = match ds.len() {
+ 1 => None,
+ 2 => Some(escape_char(ds.pop().unwrap())?),
+ n => {
+ return Err(PyValueError::new_err(format!(
+ "'like' expects 1 or 2 literals (pattern[, escape]),
got {n}"
+ )));
+ }
+ };
+ pb.like(&field, ds.pop().unwrap(), escape)
+ }
other => {
return Err(PyNotImplementedError::new_err(format!(
"unknown or unsupported predicate operator '{other}'"
@@ -296,6 +315,21 @@ fn with_field_context(err: PyErr, field: &str, data_type:
&DataType) -> PyErr {
})
}
+/// Extract a single-character `like` ESCAPE literal from an already-converted
+/// string `Datum`.
+fn escape_char(datum: Datum) -> PyResult<char> {
+ let Datum::String(s) = datum else {
+ return Err(PyValueError::new_err("'like' escape must be a str
literal"));
+ };
+ let mut chars = s.chars();
+ match (chars.next(), chars.next()) {
+ (Some(c), None) => Ok(c),
+ _ => Err(PyValueError::new_err(format!(
+ "'like' escape must be a single character, got {s:?}"
+ ))),
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -357,46 +391,197 @@ mod tests {
}
#[test]
- fn unsupported_operator_like_raises_not_implemented() {
+ fn unsupported_operator_not_raises_not_implemented() {
Python::attach(|py| {
let fields = test_fields();
- let dict = leaf_dict(py, "like", "name", &[]);
+ let dict = leaf_dict(py, "not", "id", &[1]);
let err = dict_to_predicate(&dict, &fields).unwrap_err();
assert!(err.is_instance_of::<PyNotImplementedError>(py));
});
}
+ // ---- string operators ----
+
+ /// Build a leaf dict with string literals.
+ fn str_leaf_dict<'py>(
+ py: Python<'py>,
+ method: &str,
+ field: &str,
+ literals: &[&str],
+ ) -> Bound<'py, PyDict> {
+ let d = PyDict::new(py);
+ d.set_item("method", method).unwrap();
+ d.set_item("field", field).unwrap();
+ let lits = PyList::empty(py);
+ for v in literals {
+ lits.append(*v).unwrap();
+ }
+ d.set_item("literals", lits).unwrap();
+ d
+ }
+
+ fn expect_leaf_op(pred: &Predicate, expected: PredicateOperator) {
+ match pred {
+ Predicate::Leaf { op, .. } => assert_eq!(*op, expected),
+ other => panic!("expected Leaf, got {other:?}"),
+ }
+ }
+
#[test]
- fn unsupported_operator_not_raises_not_implemented() {
+ fn starts_with_leaf_converts() {
Python::attach(|py| {
let fields = test_fields();
- let dict = leaf_dict(py, "not", "id", &[1]);
+ let dict = str_leaf_dict(py, "startsWith", "name", &["ab"]);
+ let pred = dict_to_predicate(&dict, &fields).unwrap();
+ expect_leaf_op(&pred, PredicateOperator::StartsWith);
+ });
+ }
+
+ #[test]
+ fn ends_with_leaf_converts() {
+ Python::attach(|py| {
+ let fields = test_fields();
+ let dict = str_leaf_dict(py, "endsWith", "name", &["ab"]);
+ let pred = dict_to_predicate(&dict, &fields).unwrap();
+ expect_leaf_op(&pred, PredicateOperator::EndsWith);
+ });
+ }
+
+ #[test]
+ fn contains_leaf_converts() {
+ Python::attach(|py| {
+ let fields = test_fields();
+ let dict = str_leaf_dict(py, "contains", "name", &["ab"]);
+ let pred = dict_to_predicate(&dict, &fields).unwrap();
+ expect_leaf_op(&pred, PredicateOperator::Contains);
+ });
+ }
+
+ #[test]
+ fn like_prefix_pattern_optimizes_to_starts_with() {
+ Python::attach(|py| {
+ let fields = test_fields();
+ let dict = str_leaf_dict(py, "like", "name", &["ab%"]);
+ let pred = dict_to_predicate(&dict, &fields).unwrap();
+ expect_leaf_op(&pred, PredicateOperator::StartsWith);
+ });
+ }
+
+ #[test]
+ fn like_residual_pattern_stays_like() {
+ Python::attach(|py| {
+ let fields = test_fields();
+ let dict = str_leaf_dict(py, "like", "name", &["a%b%c"]);
+ let pred = dict_to_predicate(&dict, &fields).unwrap();
+ expect_leaf_op(&pred, PredicateOperator::Like);
+ });
+ }
+
+ #[test]
+ fn like_accepts_backslash_escape_literal() {
+ Python::attach(|py| {
+ let fields = test_fields();
+ let dict = str_leaf_dict(py, "like", "name", &["100\\%%", "\\"]);
+ let pred = dict_to_predicate(&dict, &fields).unwrap();
+ // Escaped-wildcard patterns are not rewritten by the core's LIKE
+ // optimization; they stay as a residual Like leaf.
+ expect_leaf_op(&pred, PredicateOperator::Like);
+ });
+ }
+
+ #[test]
+ fn like_rejects_non_backslash_escape() {
+ Python::attach(|py| {
+ let fields = test_fields();
+ let dict = str_leaf_dict(py, "like", "name", &["100!%%", "!"]);
let err = dict_to_predicate(&dict, &fields).unwrap_err();
- assert!(err.is_instance_of::<PyNotImplementedError>(py));
+ assert!(err.is_instance_of::<PyValueError>(py));
});
}
#[test]
- fn unsupported_operator_not_without_field_raises_not_implemented() {
+ fn like_rejects_multi_char_escape() {
Python::attach(|py| {
let fields = test_fields();
- // 'not' with no 'field', only empty 'children': operator support
is
- // decided before any shape validation.
- let dict = PyDict::new(py);
- dict.set_item("method", "not").unwrap();
- dict.set_item("children", PyList::empty(py)).unwrap();
+ let dict = str_leaf_dict(py, "like", "name", &["a%", "ab"]);
let err = dict_to_predicate(&dict, &fields).unwrap_err();
- assert!(err.is_instance_of::<PyNotImplementedError>(py));
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn like_rejects_three_literals() {
+ Python::attach(|py| {
+ let fields = test_fields();
+ let dict = str_leaf_dict(py, "like", "name", &["a%", "\\", "x"]);
+ let err = dict_to_predicate(&dict, &fields).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn string_op_empty_pattern_folds_to_is_not_null() {
+ Python::attach(|py| {
+ let fields = test_fields();
+ for method in ["startsWith", "endsWith", "contains"] {
+ let dict = str_leaf_dict(py, method, "name", &[""]);
+ let pred = dict_to_predicate(&dict, &fields).unwrap();
+ expect_leaf_op(&pred, PredicateOperator::IsNotNull);
+ }
});
}
#[test]
- fn unsupported_operator_like_with_unknown_field_raises_not_implemented() {
+ fn string_op_on_non_string_column_raises_value_error() {
Python::attach(|py| {
let fields = test_fields();
- // 'like' with an unknown field: unsupported operator precedes
field
- // resolution, so NotImplementedError (not the unknown-field
ValueError).
- let dict = leaf_dict(py, "like", "nope", &[]);
+ for method in ["startsWith", "endsWith", "contains", "like"] {
+ let dict = str_leaf_dict(py, method, "id", &["a"]);
+ let err = dict_to_predicate(&dict, &fields).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py), "{method}");
+ }
+ });
+ }
+
+ #[test]
+ fn string_op_wrong_literal_count_raises_value_error() {
+ Python::attach(|py| {
+ let fields = test_fields();
+ for method in ["startsWith", "endsWith", "contains", "like"] {
+ let dict = str_leaf_dict(py, method, "name", &[]);
+ let err = dict_to_predicate(&dict, &fields).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py), "{method}
zero");
+ }
+ for method in ["startsWith", "endsWith", "contains"] {
+ let dict = str_leaf_dict(py, method, "name", &["a", "b"]);
+ let err = dict_to_predicate(&dict, &fields).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py), "{method}
two");
+ }
+ });
+ }
+
+ #[test]
+ fn string_op_unknown_field_raises_value_error() {
+ Python::attach(|py| {
+ let fields = test_fields();
+ // Now that string operators are supported, they follow the normal
+ // leaf path: field resolution happens first, so an unknown field
is
+ // a ValueError (not NotImplementedError as before).
+ let dict = str_leaf_dict(py, "like", "nope", &["x"]);
+ let err = dict_to_predicate(&dict, &fields).unwrap_err();
+ assert!(err.is_instance_of::<PyValueError>(py));
+ });
+ }
+
+ #[test]
+ fn unsupported_operator_not_without_field_raises_not_implemented() {
+ Python::attach(|py| {
+ let fields = test_fields();
+ // 'not' with no 'field', only empty 'children': operator support
is
+ // decided before any shape validation.
+ let dict = PyDict::new(py);
+ dict.set_item("method", "not").unwrap();
+ dict.set_item("children", PyList::empty(py)).unwrap();
let err = dict_to_predicate(&dict, &fields).unwrap_err();
assert!(err.is_instance_of::<PyNotImplementedError>(py));
});
@@ -429,7 +614,7 @@ mod tests {
Python::attach(|py| {
let fields = test_fields();
let ok = leaf_dict(py, "equal", "id", &[1]);
- let bad = leaf_dict(py, "like", "name", &[]);
+ let bad = leaf_dict(py, "not", "name", &[]);
let children = PyList::empty(py);
children.append(ok).unwrap();
children.append(bad).unwrap();
diff --git a/bindings/python/tests/test_read.py
b/bindings/python/tests/test_read.py
index 6d0f8f0..8065746 100644
--- a/bindings/python/tests/test_read.py
+++ b/bindings/python/tests/test_read.py
@@ -174,13 +174,12 @@ def test_filter_bool_literal_converts():
assert plan is not None
[email protected]("method", ["like", "startsWith", "not"])
-def test_filter_unsupported_operator_raises(method):
+def test_filter_unsupported_operator_raises():
with tempfile.TemporaryDirectory() as warehouse:
table = _make_table_with_data(warehouse)
with pytest.raises(NotImplementedError):
table.new_read_builder().with_filter(
- {"method": method, "field": "name", "literals": ["x"]})
+ {"method": "not", "field": "name", "literals": ["x"]})
def test_filter_unsupported_operator_precedes_shape_errors():
@@ -190,9 +189,92 @@ def
test_filter_unsupported_operator_precedes_shape_errors():
# 'not' with no field -> NotImplementedError, not ValueError about
missing field
with pytest.raises(NotImplementedError):
b.with_filter({"method": "not", "children": []})
- # 'like' with unknown field -> NotImplementedError, not ValueError
about unknown field
- with pytest.raises(NotImplementedError):
- b.with_filter({"method": "like", "field": "nope", "literals":
["x"]})
+
+
+def _make_string_table(warehouse):
+ ctx = SQLContext()
+ ctx.register_catalog("paimon", {"warehouse": warehouse})
+ ctx.sql("CREATE SCHEMA paimon.sdb")
+ ctx.sql("CREATE TABLE paimon.sdb.st (id INT, name STRING)")
+ # Two separate INSERTs -> two files, so file stats can prune per-file.
+ ctx.sql("INSERT INTO paimon.sdb.st VALUES (1, 'apple'), (2, 'apricot')")
+ ctx.sql("INSERT INTO paimon.sdb.st VALUES (3, 'banana'), (4, 'cherry')")
+ return PaimonCatalog({"warehouse": warehouse}).get_table("sdb.st")
+
+
+def _read_ids(builder):
+ splits = builder.new_scan().plan().splits()
+ batches = builder.new_read().read(splits)
+ if not batches:
+ return []
+ return sorted(pa.Table.from_batches(batches).column("id").to_pylist())
+
+
+def test_filter_starts_with_prunes_and_reads():
+ with tempfile.TemporaryDirectory() as warehouse:
+ table = _make_string_table(warehouse)
+ b = table.new_read_builder().with_filter(
+ {"method": "startsWith", "field": "name", "literals": ["ap"]})
+ assert len(b.new_scan().plan().splits()) == 1
+ assert _read_ids(b) == [1, 2]
+
+
+def test_filter_ends_with_reads_matching_rows():
+ with tempfile.TemporaryDirectory() as warehouse:
+ table = _make_string_table(warehouse)
+ b = table.new_read_builder().with_filter(
+ {"method": "endsWith", "field": "name", "literals": ["y"]})
+ assert _read_ids(b) == [4]
+
+
+def test_filter_contains_reads_matching_rows():
+ with tempfile.TemporaryDirectory() as warehouse:
+ table = _make_string_table(warehouse)
+ b = table.new_read_builder().with_filter(
+ {"method": "contains", "field": "name", "literals": ["an"]})
+ assert _read_ids(b) == [3]
+
+
+def test_filter_like_prefix_reads_matching_rows():
+ with tempfile.TemporaryDirectory() as warehouse:
+ table = _make_string_table(warehouse)
+ b = table.new_read_builder().with_filter(
+ {"method": "like", "field": "name", "literals": ["ap%"]})
+ assert _read_ids(b) == [1, 2]
+
+
+def test_filter_like_residual_pattern_reads_matching_rows():
+ with tempfile.TemporaryDirectory() as warehouse:
+ table = _make_string_table(warehouse)
+ # '_' single-char wildcard is not rewritten; exercises the Like
evaluator.
+ b = table.new_read_builder().with_filter(
+ {"method": "like", "field": "name", "literals": ["b_nana"]})
+ assert _read_ids(b) == [3]
+
+
+def test_filter_like_escape_literal():
+ with tempfile.TemporaryDirectory() as warehouse:
+ table = _make_string_table(warehouse)
+ b = table.new_read_builder()
+ # Optional second literal is the ESCAPE character; only '\\' is
accepted.
+ assert b.with_filter(
+ {"method": "like", "field": "name",
+ "literals": ["100\\%%", "\\"]}).new_scan().plan() is not None
+ with pytest.raises(ValueError):
+ b.with_filter(
+ {"method": "like", "field": "name", "literals": ["100!%%",
"!"]})
+ with pytest.raises(ValueError):
+ b.with_filter(
+ {"method": "like", "field": "name", "literals": ["a%", "ab"]})
+
+
+def test_filter_string_op_on_non_string_column_raises():
+ with tempfile.TemporaryDirectory() as warehouse:
+ table = _make_string_table(warehouse)
+ b = table.new_read_builder()
+ for method in ["startsWith", "endsWith", "contains", "like"]:
+ with pytest.raises(ValueError):
+ b.with_filter({"method": method, "field": "id", "literals":
["a"]})
def test_filter_unsupported_type_raises():
@@ -258,7 +340,9 @@ def test_filter_compound_with_unsupported_child_fails():
table = _make_table_with_data(warehouse)
pred = {"method": "and", "children": [
{"method": "equal", "field": "id", "literals": [1]},
- {"method": "like", "field": "name", "literals": ["a%"]},
+ {"method": "not", "children": [
+ {"method": "equal", "field": "name", "literals": ["a"]},
+ ]},
]}
with pytest.raises(NotImplementedError):
table.new_read_builder().with_filter(pred)
