This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new fce0633f73 [python] Support date/time extraction transforms in
query-auth rules (#10180)
fce0633f73 is described below
commit fce0633f73b419a1391cd05b6d391f3c8bc854c9
Author: jackylee <[email protected]>
AuthorDate: Fri Sep 25 21:08:53 2026 +0800
[python] Support date/time extraction transforms in query-auth rules
(#10180)
---
.../pypaimon/common/predicate_json_parser.py | 30 +++++++++++++
.../pypaimon/tests/predicate_json_parser_test.py | 49 ++++++++++++++++++++++
2 files changed, 79 insertions(+)
diff --git a/paimon-python/pypaimon/common/predicate_json_parser.py
b/paimon-python/pypaimon/common/predicate_json_parser.py
index 66e79c6881..ba895449ce 100644
--- a/paimon-python/pypaimon/common/predicate_json_parser.py
+++ b/paimon-python/pypaimon/common/predicate_json_parser.py
@@ -43,6 +43,20 @@ _TRIM_OPS = {
"TRAILING": (pc.utf8_rtrim, str.rstrip),
}
+# Calendar-field extractions on a DATE / TIMESTAMP field, mirroring Java's
+# DateExtractTransform subclasses (name -> the Arrow kernel over a timestamp).
+# QUARTER matches (month - 1) / 3 + 1; the rest map one to one.
+_DATE_EXTRACT = {
+ "YEAR": pc.year,
+ "MONTH": pc.month,
+ "DAY": pc.day,
+ "HOUR": pc.hour,
+ "MINUTE": pc.minute,
+ "SECOND": pc.second,
+ "QUARTER": pc.quarter,
+ "DAY_OF_YEAR": pc.day_of_year,
+}
+
def parse_predicate_to_batch_filter(json_str: str) ->
Callable[[pa.RecordBatch], pa.Array]:
data = json.loads(json_str)
@@ -135,9 +149,25 @@ def _apply_predicate_transform(transform: dict, batch:
pa.RecordBatch,
elif name == "NULL":
return pa.nulls(len(batch), type=null_type)
+ elif name in _DATE_EXTRACT:
+ return _date_extract(name, transform["fieldRef"], batch)
+
raise ValueError(f"Unknown transform type: {name}")
+def _date_extract(name: str, field_ref: dict, batch: pa.RecordBatch) ->
pa.Array:
+ """Extract a calendar field from a DATE / TIMESTAMP column, as INT.
+
+ A DATE is read at the start of its day (Java's LocalDate.atStartOfDay),
+ so it is cast to a timestamp first and HOUR/MINUTE/SECOND come out as 0.
+ Nulls propagate. Mirrors Java DateExtractTransform.
+ """
+ column = _field_column(field_ref, batch)
+ if pa.types.is_date(column.type):
+ column = pc.cast(column, pa.timestamp("us"))
+ return pc.cast(_DATE_EXTRACT[name](column), pa.int32())
+
+
def _substring(inputs, batch: pa.RecordBatch) -> pa.Array:
if not isinstance(inputs, list):
raise ValueError(f"SUBSTRING inputs must be a list, got {inputs!r}")
diff --git a/paimon-python/pypaimon/tests/predicate_json_parser_test.py
b/paimon-python/pypaimon/tests/predicate_json_parser_test.py
index dcc47c080c..e6f6f0d4a5 100644
--- a/paimon-python/pypaimon/tests/predicate_json_parser_test.py
+++ b/paimon-python/pypaimon/tests/predicate_json_parser_test.py
@@ -23,6 +23,7 @@ import unittest
import pyarrow as pa
from pypaimon.common.predicate_json_parser import (
+ _apply_predicate_transform,
_convert_literal,
_paimon_type_to_arrow,
extract_referenced_fields,
@@ -802,3 +803,51 @@ class TestConcatWsAllNull(unittest.TestCase):
fn = parse_predicate_to_batch_filter(pred_json)
result = fn(batch).to_pylist()
self.assertEqual(result, [False, True, False, False])
+
+
+class TestDateExtractTransforms(unittest.TestCase):
+ """YEAR / MONTH / DAY / HOUR / MINUTE / SECOND / QUARTER / DAY_OF_YEAR,
+ mirroring Java's DateExtractTransform subclasses."""
+
+ def _ts_batch(self):
+ # 2024-01-01 00:00:00 and 2024-07-01 02:24:05 (us), plus null.
+ return pa.RecordBatch.from_pydict({
+ "t": pa.array([1704067200000000, 1719800645000000, None],
+ type=pa.timestamp("us")),
+ })
+
+ def _apply(self, name, batch, field="t", ftype="TIMESTAMP(6)"):
+ transform = {"name": name,
+ "fieldRef": {"index": 0, "name": field, "type": ftype}}
+ return _apply_predicate_transform(transform, batch).to_pylist()
+
+ def test_timestamp_parts(self):
+ batch = self._ts_batch()
+ self.assertEqual(self._apply("YEAR", batch), [2024, 2024, None])
+ self.assertEqual(self._apply("MONTH", batch), [1, 7, None])
+ self.assertEqual(self._apply("DAY", batch), [1, 1, None])
+ self.assertEqual(self._apply("HOUR", batch), [0, 2, None])
+ self.assertEqual(self._apply("MINUTE", batch), [0, 24, None])
+ self.assertEqual(self._apply("SECOND", batch), [0, 5, None])
+ self.assertEqual(self._apply("QUARTER", batch), [1, 3, None])
+ self.assertEqual(self._apply("DAY_OF_YEAR", batch), [1, 183, None])
+
+ def test_date_field_starts_at_midnight(self):
+ # epoch day 19723 == 2024-01-01; a DATE has no time, so HOUR is 0.
+ batch = pa.RecordBatch.from_pydict(
+ {"d": pa.array([19723, None], type=pa.date32())})
+ self.assertEqual(self._apply("YEAR", batch, "d", "DATE"), [2024, None])
+ self.assertEqual(self._apply("MONTH", batch, "d", "DATE"), [1, None])
+ self.assertEqual(self._apply("HOUR", batch, "d", "DATE"), [0, None])
+
+ def test_wired_through_a_leaf_filter(self):
+ pred = json.dumps({
+ "kind": "LEAF",
+ "transform": {"name": "MONTH",
+ "fieldRef": {"index": 0, "name": "t", "type":
"TIMESTAMP(6)"}},
+ "function": "EQUAL",
+ "literals": [7],
+ })
+ self.assertEqual(
+
parse_predicate_to_batch_filter(pred)(self._ts_batch()).to_pylist(),
+ [False, True, False])
